fix(gui): Escape calls off a path lookup still running

Escape closed the path field, but the lookup behind it carried on: the
tab still went to the path once it answered, or alerted that it wasn't
found, and a path typed into the reopened field was ignored until then.

.github/workflows/notify-dev.yaml

@@ -0,0 +1,48 @@
+name: Notify HeyPuter (dev)
+
+on:
+  workflow_dispatch:
+    inputs:
+      puter_ref:
+        description: "puter branch, tag or SHA"
+        type: string
+        default: "main"
+
+permissions:
+  contents: read
+
+jobs:
+  authorize:
+    if: github.repository == 'HeyPuter/puter'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require maintain or admin
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ACTOR: ${{ github.triggering_actor }}
+        run: |
+          ROLE=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$ACTOR/permission" --jq .role_name)
+          if [ "$ROLE" != "maintain" ] && [ "$ROLE" != "admin" ]; then
+            echo "::error::$ACTOR has role '$ROLE'; maintain or admin required"
+            exit 1
+          fi
+
+  notify:
+    needs: authorize
+    runs-on: ubuntu-latest
+    # Holds HEYPUTER_DISPATCH_TOKEN; limited to main so branch copies of this file can't read it.
+    environment: heyputer-dispatch
+    steps:
+      - name: Trigger heyputer dev deploy
+        env:
+          PUTER_REF: ${{ inputs.puter_ref }}
+          TOKEN: ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}
+        run: |
+          jq -nc --arg ref "$PUTER_REF" \
+            '{event_type: "puter-dev-requested", client_payload: {puter_ref: $ref}}' |
+          curl -fsS -X POST \
+            -H "Authorization: token $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/HeyPuter/heyputer/dispatches \
+            -d @-
+          echo "Requested \`$PUTER_REF\`" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/notify-prod.yaml

@@ -8,6 +8,7 @@ on:
 jobs:
   notify:
     runs-on: ubuntu-latest
+    environment: heyputer-dispatch
     steps:
       - name: Trigger heyputer build
         run: |
@@ -15,4 +16,4 @@ jobs:
             -H "Authorization: token ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}" \
             -H "Accept: application/vnd.github.v3+json" \
             https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-            -d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
+            -d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'

package-lock.json

@@ -9555,9 +9555,9 @@
       "license": "Unlicense"
     },
     "node_modules/fast-uri": {
-      "version": "3.1.8",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
-      "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
+      "version": "3.1.9",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz",
+      "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==",
       "funding": [
         {
           "type": "github",
@@ -9869,9 +9869,9 @@
       "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
-      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
+      "version": "1.16.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.2.tgz",
+      "integrity": "sha512-/kVP2U8z596Ia0gwFrwyIeo74ta5aVmSeU1FfAUMAmcEP5EP7zydmag3SLiN64NaRKef3lxsq7i/g7ltJQDGow==",
       "funding": [
         {
           "type": "individual",
@@ -14244,10 +14244,13 @@
       }
     },
     "node_modules/postal-mime": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-3.0.1.tgz",
-      "integrity": "sha512-N86EnYOxZjW1uHQIgSXnnRdnJz+wxmtDsnk3C4R/9rQ1utj8A2znauXZ3Q3+stCHxG5ckY4/NrO09WiPU6rcdQ==",
-      "license": "MIT-0"
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-4.2.0.tgz",
+      "integrity": "sha512-QG5Rh8wY/zjwFF1lKWcBbwWcAWR5jWodmMPL4gMik43XDLebtWoAAPyG0lrE2kYYZ5IHTASrepGuGO0YIwNKAg==",
+      "license": "MIT-0",
+      "engines": {
+        "node": ">=20"
+      }
     },
     "node_modules/postcss": {
       "version": "8.5.26",
@@ -14525,9 +14528,9 @@
       "link": true
     },
     "node_modules/qs": {
-      "version": "6.16.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",
-      "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==",
+      "version": "6.17.1",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.17.1.tgz",
+      "integrity": "sha512-jd3COkaKVgGUW+ZHyEHCG9bH7RoaaiWuMgzsIBjUMVjLAaWGWpoUAVwljVvBHxRu/2qEHnadLGZrkutYUBPQJA==",
       "license": "BSD-3-Clause",
       "dependencies": {
         "es-define-property": "^1.0.1",
@@ -15996,9 +15999,9 @@
       }
     },
     "node_modules/systeminformation": {
-      "version": "5.33.1",
-      "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.33.1.tgz",
-      "integrity": "sha512-DEN6ICHk3Tk0Uf/hrAHh7xlt7iL5CJFBtPZinA0H62DrGG/KPKqq/Nzj6lCXPS4Ay/sf/14zNnk9LpqKzBIc+w==",
+      "version": "5.33.19",
+      "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.33.19.tgz",
+      "integrity": "sha512-Zm7wGeCG7PqfvzegMJIzrxICopWqYCeLATVmvxOOci1isLcNVKgY4Fr0hhYQzvc7cgtK3377jQVChTFAnznOiA==",
       "license": "MIT",
       "os": [
         "darwin",
@@ -17999,16 +18002,6 @@
         }
       }
     },
-    "node_modules/wrangler/node_modules/undici": {
-      "version": "7.29.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
-      "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=20.18.1"
-      }
-    },
     "node_modules/wrangler/node_modules/workerd": {
       "version": "1.20260915.1",
       "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260915.1.tgz",
@@ -18311,7 +18304,7 @@
         "openai": "^6.34.0",
         "otpauth": "^9.5.2",
         "pg": "^8.21.0",
-        "postal-mime": "^3.0.1",
+        "postal-mime": "^4.2.0",
         "replicate": "^1.0.0",
         "sharp": "^0.35.5",
         "smtp-server": "^3.19.24",
@@ -19130,7 +19123,7 @@
         "@heyputer/kv.js": "^0.2.1",
         "open": "^10.2.0",
         "path-browserify": "1.0.1",
-        "postal-mime": "^3.0.1",
+        "postal-mime": "^4.2.0",
         "socket.io-client": "^4.8.4"
       },
       "devDependencies": {

src/backend/clients/database/DatabaseClient.test.ts

@@ -285,6 +285,18 @@ describe('AbstractDatabaseClient — SQL generation', () => {
         );
     });
 
+    it('reads the database clock as epoch milliseconds in each dialect', () => {
+        expect(sqlite().nowMsExpr()).toBe(
+            "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+        );
+        expect(mysql().nowMsExpr()).toBe(
+            "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        );
+        expect(postgres().nowMsExpr()).toBe(
+            'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+        );
+    });
+
     it('describes a stored timestamp the way each driver reads it back', () => {
         const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
         expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');

src/backend/clients/database/DatabaseClient.ts

@@ -401,6 +401,21 @@ export class AbstractDatabaseClient extends PuterClient {
         });
     }
 
+    /**
+     * The database's own clock as epoch milliseconds, so nodes whose clocks
+     * disagree still agree on it. The mysql form ignores the session time zone,
+     * so a DST change can't shift it.
+     */
+    nowMsExpr(): string {
+        return this.case({
+            sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+            postgres:
+                'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+            otherwise:
+                "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        });
+    }
+
     /**
      * What a read of a timestamp column holding `seconds` returns: sqlite gives
      * back the UTC text it stored, mysql2 and pg a `Date` read in node's time

src/backend/clients/database/SqliteDatabaseClient.test.ts

@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
 import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
 
 /** Highest schema version the migration table can reach. */
-const CURRENT_SCHEMA_VERSION = 95;
+const CURRENT_SCHEMA_VERSION = 96;
 
 /**
  * These suites migrate real files on disk. Idle they finish in well under a
@@ -724,6 +724,17 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         ]);
     });
 
+    it('keeps notification.send_key unique', async () => {
+        const rows = await client.read('PRAGMA index_list(`notification`)');
+        expect(
+            rows.find((r) => r.name === 'idx_notification_send_key'),
+        ).toMatchObject({ unique: 1 });
+        const columns = await client.read(
+            'PRAGMA index_info(`idx_notification_send_key`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
+    });
+
     it('leaves an already-migrated database untouched on a second boot', async () => {
         const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
         const path = join(dir, 'nested', 'puter.sqlite');

src/backend/clients/database/SqliteDatabaseClient.ts

@@ -134,6 +134,7 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
     [92, ['0097_user-to-user-holder-permission-index.sql']],
     [93, ['0098_notification-groups.sql']],
     [94, ['0099_kv-share-handle-notify.sql']],
+    [95, ['0100_notification-send-key.sql']],
 ];
 
 export class SqliteDatabaseClient extends AbstractDatabaseClient {

src/backend/clients/database/migrations/mysql/mysql_mig_54.sql

@@ -0,0 +1,43 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
+-- rationale. The key is a uuid, so ascii and compared byte for byte. No
+-- per-file applied-state tracking, so every step is guarded and a replay does
+-- nothing.
+
+CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_send_key_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_send_key'
+  ) THEN
+    ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
+      (`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_send_key_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;

src/backend/clients/database/migrations/postgres/postgres_mig_43.sql

@@ -0,0 +1,24 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
+-- rationale. Idempotent via IF NOT EXISTS.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
+    ON notification (send_key);

src/backend/clients/database/migrations/sqlite/0100_notification-send-key.sql

@@ -0,0 +1,27 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`: the key of the delayed send (a `deferred`
+-- schedule) that wrote this row or last replaced it, derived from the
+-- schedule and its slot. Unique, so one slot can never write two rows, and a
+-- send that crashed after writing finds its row again by it. Null on every row
+-- a delayed send never touched.
+
+ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
+    ON `notification` (`send_key`);

src/backend/clients/event/types.ts

@@ -323,6 +323,8 @@ export type EventMap = {
         // the abuse extension cap sends per device (across accounts) so a
         // device farm can't dodge the per-account cap with fresh signups.
         device_fingerprint: string | null;
+        // Browser-signals dispatch id the client sent, or null.
+        dispatch_id: string | null;
         allowed: boolean;
         reason: string | null;
         // With `allowed` false, offer the card fallback in place of SMS. Only
@@ -579,6 +581,8 @@ export type EventMap = {
         updated?: true;
         /** Names this one send, for a receipt asked about it. */
         sendId: string;
+        /** The same `sendId` may be emitted again, as a delayed send is. */
+        replayable?: true;
     };
     /**
      * One notification, addressed at whatever sockets each region holds for the
@@ -950,6 +954,9 @@ export type AiCostFactorEvent = {
  */
 export type LifecyclePhase = 'before' | 'after' | 'error' | 'reject';
 
+/** Status a vetoing `before` listener may answer with. */
+export type LifecycleRejectStatus = 403 | 429;
+
 /**
  * Payload for `driver.<iface>.<method>.<phase>` events.
  *
@@ -979,6 +986,8 @@ export type DriverMethodLifecycleEvent = {
     allow?: boolean;
     /** Optional human-readable reason surfaced to the caller when vetoed. */
     rejectReason?: string;
+    /** Status for a veto: 403 (default) or 429 for a throttle. */
+    rejectStatus?: LifecycleRejectStatus;
 };
 
 /**
@@ -1013,6 +1022,8 @@ export type RouteLifecycleEvent = {
     allow?: boolean;
     /** Optional human-readable reason surfaced to the caller when vetoed. */
     rejectReason?: string;
+    /** Status for a veto: 403 (default) or 429 for a throttle. */
+    rejectStatus?: LifecycleRejectStatus;
 };
 
 export type EventKey = keyof EventMap & string;
@@ -1021,10 +1032,11 @@ export type EventKey = keyof EventMap & string;
 // Generates a wildcard for every non-final dot-separated prefix of K.
 export type WildcardPrefixes<K extends string> =
     K extends `${infer Head}.${infer Tail}`
-        ? | `${Head}.*`
-          | (Tail extends `${string}.${string}`
-                ? `${Head}.${WildcardPrefixes<Tail>}`
-                : never)
+        ?
+              | `${Head}.*`
+              | (Tail extends `${string}.${string}`
+                    ? `${Head}.${WildcardPrefixes<Tail>}`
+                    : never)
         : never;
 
 export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
@@ -1035,7 +1047,10 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
 
 /** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
 export type NotifDeliveryOutcome =
-    'received' | 'unreceived' | 'no-socket' | 'remote-only';
+    | 'received'
+    | 'unreceived'
+    | 'no-socket'
+    | 'remote-only';
 
 /** One socket's receipt for a live notification. */
 export interface NotifDeliveryResponder {

src/backend/clients/events/EventsWorkerInvokerClient.test.ts

@@ -28,13 +28,16 @@ import type { IConfig } from '../../types.js';
 import {
     DispatcherInvokeTransport,
     EVENTS_DEPLOYED_HEADER,
+    EVENTS_DISPATCH_BATCH_PATH,
     EVENTS_DISPATCH_ERROR_HEADER,
     EVENTS_DISPATCH_PATH,
     EVENTS_ERROR_HEADER,
     EVENTS_HANDLED_HEADER,
     EventsWorkerInvokerClient,
     INVOKE_CONNECTIONS,
+    SCHEDULE_INVOKE_CONNECTIONS,
     outcomeForStatus,
+    parseAnswerLine,
     type WorkerInvokeRequest,
 } from './EventsWorkerInvokerClient.js';
 
@@ -73,10 +76,7 @@ const refusedFetch: FetchImpl = (async () => {
 }) as unknown as FetchImpl;
 
 /** A stub fetch that records the URL it was called with. */
-const capturingFetch = (
-    calls: string[],
-    status = 200,
-): FetchImpl =>
+const capturingFetch = (calls: string[], status = 200): FetchImpl =>
     (async (url: string) => {
         calls.push(url);
         return new Response(null, { status });
@@ -84,9 +84,13 @@ const capturingFetch = (
 
 describe('DispatcherInvokeTransport', () => {
     it('reports a handled 400 with no error — the handler`s own refusal', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 400,
             handled: true,
@@ -94,9 +98,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('reports an unmarked 404 with no dispatch-error header as unhandled', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(404),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(404),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 404,
             handled: false,
@@ -104,9 +112,15 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('turns a dispatch-error header into a null status with the reason', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(502, { [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(502, {
+                    [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed',
+                }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: null,
             error: 'dispatcher: deploy-failed (502)',
@@ -120,9 +134,13 @@ describe('DispatcherInvokeTransport', () => {
             headers.push(init?.headers as Record<string, string>);
             return new Response(null, { status: 200 });
         }) as unknown as FetchImpl;
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl,
+            },
+        );
 
         await transport.send(CALL);
         await transport.send({ ...CALL, deployed: true });
@@ -132,9 +150,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('reports a handled 200 as settled with no error', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 200,
             handled: true,
@@ -142,9 +164,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('passes a 429 through as-is', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(429),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(429),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 429,
             handled: false,
@@ -152,18 +178,26 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('answers null and says so when the request times out', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: hangingFetch,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: hangingFetch,
+            },
+        );
         const result = await transport.send({ ...CALL, timeoutMs: 10 });
         expect(result.status).toBeNull();
         expect(result.timedOut).toBe(true);
     });
 
     it('answers null without a timeout when the request fails outright', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: refusedFetch,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: refusedFetch,
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: null,
             error: 'fetch failed',
@@ -187,12 +221,16 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('surfaces the runtime`s own error header alongside its status', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(500, {
-                [EVENTS_HANDLED_HEADER]: '1',
-                [EVENTS_ERROR_HEADER]: 'handler-threw',
-            }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(500, {
+                    [EVENTS_HANDLED_HEADER]: '1',
+                    [EVENTS_ERROR_HEADER]: 'handler-threw',
+                }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 500,
             handled: true,
@@ -208,7 +246,9 @@ describe('DispatcherInvokeTransport', () => {
             { fetchImpl: capturingFetch(calls) },
         );
         await transport.send(CALL);
-        expect(calls).toEqual([`http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`]);
+        expect(calls).toEqual([
+            `http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`,
+        ]);
     });
 });
 
@@ -237,10 +277,12 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
     /** Every connection taken by a call nobody has answered yet. */
     const saturated = async () => {
         const { fetchImpl, held } = heldFetch();
+        // Slots are per call, batched or not; one request per call keeps
+        // the count readable.
         const transport = new DispatcherInvokeTransport(
             'http://dispatcher',
             's',
-            { fetchImpl },
+            { fetchImpl, batchSize: 1 },
         );
         const busy = Array.from({ length: INVOKE_CONNECTIONS }, () =>
             transport.send(CALL),
@@ -303,6 +345,534 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
     });
 });
 
+describe('DispatcherInvokeTransport schedule lane', () => {
+    const heldFetch = () => {
+        const held: Array<{ answer: () => void; body: string }> = [];
+        const fetchImpl = ((_url: string, init: RequestInit) =>
+            new Promise((resolve) => {
+                held.push({
+                    body: String(init.body),
+                    answer: () =>
+                        resolve(
+                            new Response(null, {
+                                status: 200,
+                                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                            }),
+                        ),
+                });
+            })) as unknown as FetchImpl;
+        return { fetchImpl, held };
+    };
+
+    it('holds schedule runs to their share and keeps serving deliveries', async () => {
+        const { fetchImpl, held } = heldFetch();
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl, batchSize: 1 },
+        );
+        const scheduled = Array.from(
+            { length: SCHEDULE_INVOKE_CONNECTIONS + 5 },
+            () => transport.send({ ...CALL, lane: 'schedule' as const }),
+        );
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
+        );
+        // A delivery waiting behind the schedule queue still gets a connection.
+        const delivery = transport.send(CALL);
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
+        );
+        held[SCHEDULE_INVOKE_CONNECTIONS].answer();
+        await expect(delivery).resolves.toMatchObject({ status: 200 });
+
+        // A schedule run's slot goes to the next schedule run waiting.
+        held[0].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 2),
+        );
+        for (let i = 0; i < 4; i++) held[i + 1].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 6),
+        );
+        for (const call of held) call.answer();
+        await Promise.all(scheduled);
+    });
+
+    it('builds the body after the connection wait, so a stamp is the send time', async () => {
+        const { fetchImpl, held } = heldFetch();
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl, batchSize: 1 },
+        );
+        const busy = Array.from({ length: SCHEDULE_INVOKE_CONNECTIONS }, () =>
+            transport.send({ ...CALL, lane: 'schedule' as const }),
+        );
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
+        );
+        let renders = 0;
+        const stamped = transport.send({
+            ...CALL,
+            lane: 'schedule',
+            render: () => `{"n":${++renders}}`,
+        });
+        held[0].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
+        );
+        expect(held[SCHEDULE_INVOKE_CONNECTIONS].body).toBe('{"n":1}');
+        for (const call of held) call.answer();
+        await Promise.all([...busy, stamped]);
+    });
+});
+
+describe('DispatcherInvokeTransport batches', () => {
+    interface BatchRequest {
+        url: string;
+        body: { calls: Array<Record<string, unknown>> };
+        signal: AbortSignal;
+        /** Streams one answer line back. */
+        answer: (line: Record<string, unknown>) => void;
+        end: () => void;
+    }
+
+    /**
+     * A fetch whose batch answers stream only when told to; single calls get a
+     * handled 200 straight away.
+     */
+    const streamingFetch = () => {
+        const batches: BatchRequest[] = [];
+        const singles: Array<{ url: string; headers: Record<string, string> }> =
+            [];
+        const fetchImpl = (async (url: string, init: RequestInit) => {
+            if (!url.endsWith(EVENTS_DISPATCH_BATCH_PATH)) {
+                singles.push({
+                    url,
+                    headers: init.headers as Record<string, string>,
+                });
+                return new Response(null, {
+                    status: 200,
+                    headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                });
+            }
+            let controller!: ReadableStreamDefaultController<Uint8Array>;
+            const stream = new ReadableStream<Uint8Array>({
+                start: (c) => {
+                    controller = c;
+                },
+            });
+            const encoder = new TextEncoder();
+            init.signal?.addEventListener('abort', () => {
+                try {
+                    controller.error(init.signal!.reason);
+                } catch {
+                    // already closed
+                }
+            });
+            batches.push({
+                url,
+                body: JSON.parse(String(init.body)),
+                signal: init.signal!,
+                answer: (line) =>
+                    controller.enqueue(
+                        encoder.encode(`${JSON.stringify(line)}\n`),
+                    ),
+                end: () => controller.close(),
+            });
+            return new Response(stream, {
+                status: 200,
+                headers: { 'content-type': 'application/x-ndjson' },
+            });
+        }) as unknown as FetchImpl;
+        return { fetchImpl, batches, singles };
+    };
+
+    const transportFor = (
+        fetchImpl: FetchImpl,
+        options: { batchSize?: number; batchWindowMs?: number } = {},
+    ) =>
+        new DispatcherInvokeTransport('http://dispatcher', 's', {
+            fetchImpl,
+            ...options,
+        });
+
+    it('sends calls made together as one request, and answers each from its own line', async () => {
+        const { fetchImpl, batches, singles } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const first = transport.send({ ...CALL, script: 'evw-a', key: 'k1:a' });
+        const second = transport.send({
+            ...CALL,
+            script: 'evw-b',
+            deployed: true,
+            body: '{"b":1}',
+        });
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+
+        expect(batches[0].url).toBe(
+            `http://dispatcher${EVENTS_DISPATCH_BATCH_PATH}`,
+        );
+        expect(batches[0].body.calls).toEqual([
+            {
+                script: 'evw-a',
+                appUid: 'app-test',
+                key: 'k1:a',
+                body: '{}',
+                waitMs: expect.any(Number),
+            },
+            {
+                script: 'evw-b',
+                appUid: 'app-test',
+                key: 'k1:test',
+                deployed: true,
+                body: '{"b":1}',
+                waitMs: expect.any(Number),
+            },
+        ]);
+        // What is left of half the timeout, the share a call may wait.
+        expect(batches[0].body.calls[0].waitMs).toBeLessThanOrEqual(2_500);
+        expect(batches[0].body.calls[0].waitMs).toBeGreaterThan(2_400);
+
+        // Out of order: each line names the call it answers.
+        batches[0].answer({ i: 1, status: 400, handled: true });
+        await expect(second).resolves.toEqual({ status: 400, handled: true });
+        batches[0].answer({
+            i: 0,
+            status: 503,
+            handled: true,
+            error: 'platform-error',
+            retryAfter: '3',
+        });
+        await expect(first).resolves.toEqual({
+            status: 503,
+            handled: true,
+            error: 'platform-error',
+            retryAfterMs: 3000,
+        });
+        // Every call answered, so the request is let go.
+        expect(batches[0].signal.aborted).toBe(true);
+        expect(singles).toEqual([]);
+    });
+
+    it('sends a call with nobody to batch with on the single route', async () => {
+        const { fetchImpl, batches, singles } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        await expect(transport.send(CALL)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(batches).toEqual([]);
+        expect(singles[0].url).toBe(`http://dispatcher${EVENTS_DISPATCH_PATH}`);
+    });
+
+    it('sends a full batch at once, without waiting out the window', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl, {
+            batchSize: 2,
+            batchWindowMs: 60_000,
+        });
+        const sent = [transport.send(CALL), transport.send(CALL)];
+        const third = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        expect(batches[0].body.calls).toHaveLength(2);
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await Promise.all(sent);
+        transport.close();
+        await expect(third).resolves.toMatchObject({ status: 200 });
+    });
+
+    it('gives each call its own timeout, without failing the rest of the batch', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const quick = transport.send({ ...CALL, timeoutMs: 50 });
+        const slow = transport.send({ ...CALL, timeoutMs: 5_000 });
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, started: true });
+        batches[0].answer({ i: 1, started: true });
+
+        await expect(quick).resolves.toMatchObject({
+            status: null,
+            timedOut: true,
+        });
+        expect(batches[0].signal.aborted).toBe(false);
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await expect(slow).resolves.toEqual({ status: 200, handled: true });
+    });
+
+    it('runs a call`s timeout from its started line, not from the send', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        let settled = false;
+        const queued = transport
+            .send({ ...CALL, timeoutMs: 200 })
+            .then((answer) => {
+                settled = true;
+                return answer;
+            });
+        const other = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+
+        // Queued behind other calls for longer than its whole timeout.
+        await new Promise((resolve) => setTimeout(resolve, 250));
+        expect(settled).toBe(false);
+        batches[0].answer({ i: 0, started: true });
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        expect(settled).toBe(false);
+        await expect(queued).resolves.toMatchObject({
+            status: null,
+            timedOut: true,
+        });
+
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await other;
+    });
+
+    it('puts a call that never started on Puter, not on its handler', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const never = transport.send({ ...CALL, timeoutMs: 20 });
+        const queueTimedOut = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 1, status: 503, dispatch: 'queue-timeout' });
+
+        await expect(queueTimedOut).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: queue-timeout (503)',
+            dispatchReason: 'queue-timeout',
+        });
+        // Its wait plus the grace for the started line to arrive.
+        await expect(never).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: call never started',
+        });
+        const client = new EventsWorkerInvokerClient({} as IConfig);
+        client.setTransport({ send: async () => queueTimedOut });
+        await expect(client.invoke(REQUEST)).resolves.toMatchObject({
+            outcome: 'unavailable',
+        });
+    });
+
+    it('puts a call the stream ended without answering on Puter', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const answered = transport.send(CALL);
+        const dropped = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].end();
+
+        await expect(answered).resolves.toEqual({ status: 200, handled: true });
+        await expect(dropped).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: batch ended without an answer',
+        });
+    });
+
+    it('carries a dispatch failure in a line as that call`s reason', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const missing = transport.send(CALL);
+        const other = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+
+        await expect(missing).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: missing (404)',
+            dispatchReason: 'missing',
+        });
+        await expect(other).resolves.toMatchObject({ status: 200 });
+    });
+
+    it('fails every call of a batch the dispatcher refused, or something else answered', async () => {
+        for (const [response, expected] of [
+            [
+                new Response('forbidden', {
+                    status: 403,
+                    headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'forbidden' },
+                }),
+                {
+                    status: null,
+                    error: 'dispatcher: forbidden (403)',
+                    dispatchReason: 'forbidden',
+                },
+            ],
+            [
+                new Response('not found', { status: 404 }),
+                {
+                    status: null,
+                    error: 'dispatcher: unmarked 404 for a batch',
+                },
+            ],
+        ] as const) {
+            const transport = transportFor(
+                (async () => response) as unknown as FetchImpl,
+            );
+            const answers = await Promise.all([
+                transport.send(CALL),
+                transport.send(CALL),
+            ]);
+            expect(answers).toEqual([expected, expected]);
+        }
+    });
+
+    it('fails every call when the batch request itself fails', async () => {
+        const transport = transportFor(refusedFetch);
+        const answers = await Promise.all([
+            transport.send(CALL),
+            transport.send(CALL),
+        ]);
+        expect(answers).toEqual([
+            { status: null, error: 'fetch failed' },
+            { status: null, error: 'fetch failed' },
+        ]);
+    });
+
+    it('falls back to single calls on a dispatcher without the batch route', async () => {
+        const urls: string[] = [];
+        const fetchImpl = (async (url: string) => {
+            urls.push(url);
+            if (url.endsWith(EVENTS_DISPATCH_BATCH_PATH))
+                return new Response('no-such-route', {
+                    status: 404,
+                    headers: {
+                        [EVENTS_DISPATCH_ERROR_HEADER]: 'no-such-route',
+                    },
+                });
+            return new Response(null, {
+                status: 200,
+                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+            });
+        }) as unknown as FetchImpl;
+        const transport = transportFor(fetchImpl);
+
+        const first = await Promise.all([
+            transport.send(CALL),
+            transport.send(CALL),
+        ]);
+        expect(first).toEqual([
+            { status: 200, handled: true },
+            { status: 200, handled: true },
+        ]);
+        // The next pair goes straight to the single route.
+        await Promise.all([transport.send(CALL), transport.send(CALL)]);
+        expect(urls.map((url) => new URL(url).pathname)).toEqual([
+            EVENTS_DISPATCH_BATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+        ]);
+    });
+
+    it('builds each body when the batch goes out', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        let renders = 0;
+        const sent = [
+            transport.send({ ...CALL, render: () => `{"n":${++renders}}` }),
+            transport.send(CALL),
+        ];
+        expect(renders).toBe(0);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        expect(batches[0].body.calls[0].body).toBe('{"n":1}');
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await Promise.all(sent);
+    });
+
+    it('deploys a call`s missing script and retries it, batched with the rest', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const client = new EventsWorkerInvokerClient({
+            events: { invokeTimeoutMs: 5_000 },
+        } as unknown as IConfig);
+        client.setTransport(transportFor(fetchImpl));
+        const missHandler = vi.fn(async () => 'deployed' as const);
+        client.setMissHandler(missHandler);
+
+        const missing = client.invoke({ ...REQUEST, script: 'evw-gone' });
+        const resident = client.invoke(REQUEST);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await expect(resident).resolves.toMatchObject({ outcome: 'settled' });
+
+        // The retry is alone, so it goes on the single route.
+        await expect(missing).resolves.toMatchObject({ outcome: 'settled' });
+        expect(missHandler).toHaveBeenCalledWith('app-test', 'evw-gone', {
+            lane: undefined,
+        });
+    });
+
+    it('reads only well-formed answer lines', () => {
+        expect(parseAnswerLine('not json')).toBeNull();
+        expect(parseAnswerLine('{"status":200}')).toBeNull();
+        expect(parseAnswerLine('{"i":-1,"status":200}')).toBeNull();
+        expect(parseAnswerLine('{"i":0}')).toBeNull();
+        expect(parseAnswerLine('{"i":1,"started":true}')).toEqual({
+            i: 1,
+            started: true,
+        });
+        expect(parseAnswerLine('{"i":2,"status":200}')).toEqual({
+            i: 2,
+            answer: { status: 200, handled: false },
+        });
+    });
+});
+
+describe('EventsWorkerInvokerClient invoke options', () => {
+    it('stamps the event as each attempt goes out', async () => {
+        const bodies: string[] = [];
+        const client = makeClient();
+        client.setTransport({
+            send: async (call) => {
+                bodies.push(call.body);
+                return { status: 200, handled: true };
+            },
+        });
+        let n = 0;
+        await client.invoke(REQUEST, {
+            stampEvent: (event) => ({ ...(event as object), sentAt: ++n }),
+        });
+        expect(JSON.parse(bodies[0]).event).toEqual({ sentAt: 1 });
+    });
+
+    it('gives a call up as Puter`s when its deploy is put off for another', async () => {
+        const headersSeen: Array<Record<string, string>> = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch(headersSeen),
+            }),
+        );
+        client.setMissHandler(async () => 'busy');
+        await expect(client.invoke(REQUEST)).resolves.toMatchObject({
+            outcome: 'unavailable',
+            deployBusy: true,
+        });
+        expect(headersSeen).toHaveLength(1);
+    });
+
+    it('tells the miss handler whose call needs the deploy', async () => {
+        const lanes: unknown[] = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch([]),
+            }),
+        );
+        client.setMissHandler(async (_appUid, _script, options) => {
+            lanes.push(options.lane);
+            return 'deployed';
+        });
+        await client.invoke(REQUEST, { lane: 'schedule' });
+        await client.invoke(REQUEST);
+        expect(lanes).toEqual(['schedule', undefined]);
+    });
+});
+
 const REQUEST: WorkerInvokeRequest = {
     script: 'evw-test',
     appUid: 'app-test',
@@ -313,7 +883,8 @@ const REQUEST: WorkerInvokeRequest = {
     ctx: {},
 };
 
-const makeClient = () => new EventsWorkerInvokerClient({} as unknown as IConfig);
+const makeClient = () =>
+    new EventsWorkerInvokerClient({} as unknown as IConfig);
 
 /** Answers `missing` until it sees the deployed header, then settles. */
 const missUntilDeployedFetch = (

src/backend/clients/events/EventsWorkerInvokerClient.ts

@@ -43,6 +43,20 @@ import { PuterClient } from '../types.js';
  *
  * On a miss, a handler set here may deploy the script itself and retry once
  * with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
+ *
+ * Calls that come due within a few milliseconds of each other go out together:
+ *
+ *     POST <events dispatcher>/invoke-batch
+ *     x-puter-internal-auth:  <internal secret>
+ *     { calls: [{ script, appUid, key, deployed?, body, waitMs }] }
+ *
+ * The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
+ * started: true }` when a call goes to its worker, then `{ i, status, handled?,
+ * error?, retryAfter?, dispatch? }` when it finishes — the same facts the
+ * single route carries in its status and headers. A call's timeout runs from
+ * its `started` line, so time spent queued behind other calls is never the
+ * handler's. A call still queued `waitMs` after the batch arrived is never
+ * started and answers `dispatch: "queue-timeout"`.
  */
 
 /** The one route an events worker answers. */
@@ -51,6 +65,27 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
 /** The route the events dispatcher takes an invocation on. */
 export const EVENTS_DISPATCH_PATH = '/invoke';
 
+/** The route it takes several on, answering each as it finishes. */
+export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
+
+/** Calls one batch carries at most. */
+export const INVOKE_BATCH_SIZE = 60;
+
+/** How long the first call of a batch waits for others to join it. */
+export const INVOKE_BATCH_WINDOW_MS = 5;
+
+/**
+ * How long a dispatcher without the batch route is sent single calls before the
+ * batch route is tried again.
+ */
+const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
+
+/**
+ * How long past its wait a batched call's `started` line may take to arrive
+ * before the call is given up as never started.
+ */
+const BATCH_START_GRACE_MS = 2_000;
+
 /**
  * Header the dispatcher marks its own failures with. Anything carrying it never
  * reached a handler, so its status says nothing about the delivery.
@@ -112,10 +147,16 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
  */
 const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
 
+/** Whose call asked for the deploy, for the share of deploys it may take. */
+export interface WorkerMissOptions {
+    lane?: InvokeLane;
+}
+
 /** What a self-deploy attempt did, and to which app/script it applies. */
 export type WorkerMissHandler = (
     appUid: string,
     script: string,
+    options: WorkerMissOptions,
 ) => Promise<'deployed' | string>;
 
 /** How long a handler has to answer before the attempt is abandoned. */
@@ -127,6 +168,18 @@ export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
  * that waits for a free one before its invoke timeout starts.
  */
 export const INVOKE_CONNECTIONS = 256;
+
+/**
+ * Of those, how many schedule runs may hold at once, so a burst of schedules
+ * can never take the connections event deliveries need.
+ */
+export const SCHEDULE_INVOKE_CONNECTIONS = 64;
+
+/** Whose call this is, for the connection share it may take. */
+export type InvokeLane = 'delivery' | 'schedule';
+
+/** What a deploy answers when another deploy of the region holds the way. */
+export const DEPLOY_BUSY = 'busy';
 const INVOKE_KEEP_ALIVE_MS = 30_000;
 
 /**
@@ -161,6 +214,12 @@ export interface EventsInvokeCall {
     timeoutMs: number;
     /** Set on the retry after this backend deployed the script itself. */
     deployed?: boolean;
+    lane?: InvokeLane;
+    /**
+     * Builds `body` again at send time, for a transport that waits for a
+     * connection first, so a send time stamped into it is the real one.
+     */
+    render?: () => string;
 }
 
 /** What one attempt at delivering an invocation came back with. */
@@ -176,6 +235,8 @@ export interface EventsInvokeAnswer {
     timedOut?: boolean;
     /** From a `retry-after` header in seconds. */
     retryAfterMs?: number;
+    /** A deploy this call needed was put off: another held the way. */
+    deployBusy?: boolean;
 }
 
 /**
@@ -194,10 +255,15 @@ export interface EventsInvokeTransport {
  * handler); retried like `retriable` but never the handler's fault.
  */
 export type WorkerInvokeOutcome =
-    'settled' | 'terminal' | 'retriable' | 'unavailable';
+    | 'settled'
+    | 'terminal'
+    | 'retriable'
+    | 'unavailable';
 
 export interface WorkerInvokeResult {
     outcome: WorkerInvokeOutcome;
+    /** A deploy on miss was put off because another one held the way. */
+    deployBusy?: boolean;
     /** The status the handler answered with, or `null` when it never did. */
     status: number | null;
     /** Why it could not answer, when nothing did. */
@@ -206,6 +272,13 @@ export interface WorkerInvokeResult {
     retryAfterMs?: number;
 }
 
+/** How one invocation is sent. */
+export interface WorkerInvokeOptions {
+    lane?: InvokeLane;
+    /** Applied to the event as each attempt is sent. */
+    stampEvent?: (event: unknown) => unknown;
+}
+
 const NO_TRANSPORT =
     'no events dispatcher configured (url and internal secret)';
 
@@ -236,7 +309,10 @@ export class EventsWorkerInvokerClient extends PuterClient {
         this.#dispatcher = null;
     }
 
-    async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
+    async invoke(
+        request: WorkerInvokeRequest,
+        options: WorkerInvokeOptions = {},
+    ): Promise<WorkerInvokeResult> {
         const transport = this.#pickTransport();
         if (!transport)
             return {
@@ -245,32 +321,41 @@ export class EventsWorkerInvokerClient extends PuterClient {
                 error: NO_TRANSPORT,
             };
 
-        const call: EventsInvokeCall = {
-            script: request.script,
-            appUid: request.appUid,
-            key: request.key,
-            body: JSON.stringify({
+        const { stampEvent } = options;
+        const render = (): string =>
+            JSON.stringify({
                 handler: request.handler,
-                event: request.event,
+                event: stampEvent ? stampEvent(request.event) : request.event,
                 ctx: request.ctx,
                 token: request.token,
-            }),
-            timeoutMs: this.#timeoutMs(),
-        };
+            });
+        const send = (deployed: boolean): Promise<EventsInvokeAnswer> =>
+            transport.send({
+                script: request.script,
+                appUid: request.appUid,
+                key: request.key,
+                body: render(),
+                timeoutMs: this.#timeoutMs(),
+                ...(deployed ? { deployed: true } : {}),
+                ...(options.lane ? { lane: options.lane } : {}),
+                ...(stampEvent ? { render } : {}),
+            });
 
-        let result = await transport.send(call);
+        let result = await send(false);
         if (this.#missHandler && selfDeployable(result)) {
             const outcome = await this.#missHandler(
                 request.appUid,
                 request.script,
+                { lane: options.lane },
             );
             if (outcome !== 'deployed')
                 return {
                     outcome: 'unavailable',
                     status: null,
                     error: `deploy: ${outcome}`,
+                    ...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
                 };
-            result = await transport.send({ ...call, deployed: true });
+            result = await send(true);
         } else if (
             result.status === null &&
             result.dispatchReason &&
@@ -278,10 +363,17 @@ export class EventsWorkerInvokerClient extends PuterClient {
         ) {
             // No deploy to trigger — just give propagation the one retry
             // it usually needs, without spending an upload on it.
-            result = await transport.send({ ...call, deployed: true });
+            result = await send(true);
         }
 
         const { status, handled, error, timedOut, retryAfterMs } = result;
+        if (result.deployBusy)
+            return {
+                outcome: 'unavailable',
+                status: null,
+                error,
+                deployBusy: true,
+            };
         // Past the timeout the handler was most likely still running, which
         // is its own doing; any other missing answer never reached it.
         if (status === null)
@@ -313,9 +405,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         // a misconfiguration worth reporting once per attempt rather than a
         // round trip.
         if (!dispatcherUrl || !internalSecret) return null;
+        const { invokeBatchSize, invokeBatchWindowMs } =
+            this.config.events ?? {};
         this.#dispatcher ??= new DispatcherInvokeTransport(
             dispatcherUrl,
             internalSecret,
+            {
+                ...(invokeBatchSize !== undefined
+                    ? { batchSize: invokeBatchSize }
+                    : {}),
+                ...(invokeBatchWindowMs !== undefined
+                    ? { batchWindowMs: invokeBatchWindowMs }
+                    : {}),
+            },
         );
         return this.#dispatcher;
     }
@@ -328,28 +430,63 @@ export class EventsWorkerInvokerClient extends PuterClient {
     }
 }
 
+/** A call waiting in, or sent with, the current batch. */
+interface BatchedCall {
+    call: EventsInvokeCall;
+    resolve: (answer: EventsInvokeAnswer) => void;
+    /** When its wait for a connection began, for what is left of it. */
+    waitingSince: number;
+}
+
+export interface DispatcherInvokeTransportOptions {
+    fetchImpl?: typeof undiciFetch;
+    /** Calls per batch; 1 sends every call on its own. */
+    batchSize?: number;
+    batchWindowMs?: number;
+}
+
 /** Reaches an events worker the only way production can: through the dispatcher. */
 export class DispatcherInvokeTransport implements EventsInvokeTransport {
     readonly #url: string;
     readonly #secret: string;
     readonly #fetchImpl: typeof undiciFetch;
+    readonly #batchSize: number;
+    readonly #batchWindowMs: number;
     #agent: Agent | null = null;
     /** Calls holding a connection slot. */
     #active = 0;
-    /** Calls waiting for one, longest-waiting first. */
-    readonly #waiting = new Set<() => void>();
+    /** Of those, schedule runs. */
+    #scheduleActive = 0;
+    /** Calls waiting for one, longest-waiting first, with their lane. */
+    readonly #waiting = new Map<() => void, InvokeLane>();
+    /** Calls gathered for the next batch. */
+    #batch: BatchedCall[] = [];
+    #batchTimer: ReturnType<typeof setTimeout> | null = null;
+    /** Until when the dispatcher is taken not to have the batch route. */
+    #batchUnsupportedUntil = 0;
 
     constructor(
         dispatcherUrl: string,
         secret: string,
-        options: { fetchImpl?: typeof undiciFetch } = {},
+        options: DispatcherInvokeTransportOptions = {},
     ) {
-        this.#url = dispatcherUrl;
+        // Joined as a string: `new URL(path, base)` resolves an absolute path
+        // against the base's origin, dropping any path prefix it carries.
+        this.#url = dispatcherUrl.replace(/\/$/, '');
         this.#secret = secret;
         this.#fetchImpl = options.fetchImpl ?? undiciFetch;
+        this.#batchSize = Math.max(
+            1,
+            Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
+        );
+        this.#batchWindowMs = Math.max(
+            0,
+            options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
+        );
     }
 
     close(): void {
+        this.#flush();
         void this.#agent?.close().catch(() => {});
         this.#agent = null;
     }
@@ -357,29 +494,68 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
         // No `timedOut`: the handler never got this call, so the wait is not
         // its failure.
-        if (!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE)))
+        const lane = call.lane ?? 'delivery';
+        const waitingSince = Date.now();
+        if (
+            !(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
+        )
             return {
                 status: null,
                 error: 'no free connection to the events dispatcher',
             };
         try {
-            // `new URL(path, base)` resolves an absolute path against the
-            // base's origin, dropping any path prefix `base` already carries.
-            const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
-            const response = await this.#fetchImpl(url, {
-                method: 'POST',
-                headers: {
-                    'content-type': 'application/json',
-                    'x-puter-internal-auth': this.#secret,
-                    'x-puter-events-script': call.script,
-                    'x-puter-events-app': call.appUid,
-                    'x-puter-events-key': call.key,
-                    ...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
-                },
-                body: call.body,
-                signal: AbortSignal.timeout(call.timeoutMs),
-                dispatcher: this.#dispatcherFor(),
+            if (this.#batchSize === 1) return await this.#sendOne(call);
+            return await new Promise<EventsInvokeAnswer>((resolve) => {
+                this.#batch.push({ call, resolve, waitingSince });
+                if (this.#batch.length >= this.#batchSize) this.#flush();
+                else
+                    this.#batchTimer ??= setTimeout(
+                        () => this.#flush(),
+                        this.#batchWindowMs,
+                    );
             });
+        } finally {
+            this.#release(lane);
+        }
+    }
+
+    /** Send what has gathered: alone when it is one call, else as a batch. */
+    #flush(): void {
+        if (this.#batchTimer) clearTimeout(this.#batchTimer);
+        this.#batchTimer = null;
+        const batch = this.#batch;
+        this.#batch = [];
+        if (batch.length === 0) return;
+        if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
+            for (const { call, resolve } of batch)
+                void this.#sendOne(call).then(resolve);
+            return;
+        }
+        void this.#sendBatch(batch);
+    }
+
+    /** One call on the single route. */
+    async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                        'x-puter-events-script': call.script,
+                        'x-puter-events-app': call.appUid,
+                        'x-puter-events-key': call.key,
+                        ...(call.deployed
+                            ? { [EVENTS_DEPLOYED_HEADER]: '1' }
+                            : {}),
+                    },
+                    body: call.render ? call.render() : call.body,
+                    signal: AbortSignal.timeout(call.timeoutMs),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
             // The body is not read: a handler answers with a status, and
             // whatever else it writes is its own business.
             void response.body?.cancel().catch(() => {});
@@ -390,24 +566,181 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 EVENTS_DISPATCH_ERROR_HEADER,
             );
             if (dispatchError !== null)
-                return {
-                    status: null,
-                    error: `dispatcher: ${dispatchError} (${response.status})`,
-                    dispatchReason: dispatchError,
-                };
+                return dispatchAnswer(dispatchError, response.status);
 
             return answerFrom(response);
         } catch (err) {
             return failedAnswer(err);
-        } finally {
-            this.#release();
         }
     }
 
+    /**
+     * Several calls in one request. A call may wait in the dispatcher's queue
+     * for whatever is left of its connection wait, and its timeout runs from
+     * its `started` line; the request itself is dropped once every call has an
+     * answer.
+     */
+    async #sendBatch(batch: BatchedCall[]): Promise<void> {
+        const hangUp = new AbortController();
+        let open = batch.length;
+        const answered = batch.map(() => false);
+        const timers: ReturnType<typeof setTimeout>[] = [];
+        const settle = (i: number, answer: EventsInvokeAnswer): void => {
+            if (answered[i]) return;
+            answered[i] = true;
+            clearTimeout(timers[i]);
+            batch[i].resolve(answer);
+            if (--open === 0) hangUp.abort();
+        };
+        const settleRest = (answer: EventsInvokeAnswer): void => {
+            for (let i = 0; i < batch.length; i++) settle(i, answer);
+        };
+
+        const now = Date.now();
+        const waits = batch.map(({ call, waitingSince }) =>
+            Math.max(
+                0,
+                Math.floor(
+                    call.timeoutMs * CONNECTION_WAIT_SHARE -
+                        (now - waitingSince),
+                ),
+            ),
+        );
+        const body = JSON.stringify({
+            calls: batch.map(({ call }, i) => ({
+                script: call.script,
+                appUid: call.appUid,
+                key: call.key,
+                ...(call.deployed ? { deployed: true } : {}),
+                body: call.render ? call.render() : call.body,
+                waitMs: waits[i],
+            })),
+        });
+        const arm = (
+            i: number,
+            ms: number,
+            answer: () => EventsInvokeAnswer,
+        ) => {
+            clearTimeout(timers[i]);
+            timers[i] = setTimeout(() => settle(i, answer()), ms);
+            timers[i].unref?.();
+        };
+        // No `timedOut`: a call that never started never reached a handler.
+        batch.forEach((_, i) =>
+            arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
+                status: null,
+                error: 'dispatcher: call never started',
+            })),
+        );
+        const started = (i: number): void => {
+            if (!answered[i])
+                arm(i, batch[i].call.timeoutMs, () =>
+                    failedAnswer(timeoutError()),
+                );
+        };
+        const longest = Math.max(
+            ...batch.map(
+                ({ call }, i) =>
+                    waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
+            ),
+        );
+
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                    },
+                    body,
+                    signal: AbortSignal.any([
+                        hangUp.signal,
+                        AbortSignal.timeout(longest),
+                    ]),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
+
+            const dispatchError = response.headers.get(
+                EVENTS_DISPATCH_ERROR_HEADER,
+            );
+            if (dispatchError === 'no-such-route') {
+                // A dispatcher from before the batch route: nothing ran, so
+                // every call goes again on its own.
+                void response.body?.cancel().catch(() => {});
+                this.#batchUnsupportedUntil =
+                    Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
+                batch.forEach(({ call }, i) => {
+                    if (!answered[i])
+                        void this.#sendOne(call).then((a) => settle(i, a));
+                });
+                return;
+            }
+            if (dispatchError !== null || !response.ok) {
+                void response.body?.cancel().catch(() => {});
+                // An unmarked failure (an edge 404, a WAF 403) reads as
+                // Puter's, never as any handler's answer.
+                settleRest(
+                    dispatchError !== null
+                        ? dispatchAnswer(dispatchError, response.status)
+                        : {
+                              status: null,
+                              error: `dispatcher: unmarked ${response.status} for a batch`,
+                          },
+                );
+                return;
+            }
+
+            let buffered = '';
+            const read = (line: string): void => {
+                if (line.trim() === '') return;
+                const parsed = parseAnswerLine(line);
+                if (!parsed || parsed.i >= batch.length) return;
+                if ('started' in parsed) started(parsed.i);
+                else settle(parsed.i, parsed.answer);
+            };
+            const decoder = new TextDecoder();
+            for await (const chunk of response.body ?? []) {
+                buffered += decoder.decode(chunk, { stream: true });
+                let newline = buffered.indexOf('\n');
+                while (newline !== -1) {
+                    read(buffered.slice(0, newline));
+                    buffered = buffered.slice(newline + 1);
+                    newline = buffered.indexOf('\n');
+                }
+            }
+            read(buffered);
+            settleRest({
+                status: null,
+                error: 'dispatcher: batch ended without an answer',
+            });
+        } catch (err) {
+            // Every call has its answer and the request was dropped on purpose.
+            if (open === 0) return;
+            settleRest(failedAnswer(err));
+        }
+    }
+
+    /** Whether a call on `lane` may take a connection now. */
+    #free(lane: InvokeLane): boolean {
+        return (
+            this.#active < INVOKE_CONNECTIONS &&
+            (lane !== 'schedule' ||
+                this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
+        );
+    }
+
+    #take(lane: InvokeLane): void {
+        this.#active++;
+        if (lane === 'schedule') this.#scheduleActive++;
+    }
+
     /** A connection slot, or false once `waitMs` passes without one. */
-    #acquire(waitMs: number): Promise<boolean> {
-        if (this.#active < INVOKE_CONNECTIONS) {
-            this.#active++;
+    #acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
+        if (this.#free(lane)) {
+            this.#take(lane);
             return Promise.resolve(true);
         }
         return new Promise((resolve) => {
@@ -420,19 +753,24 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 resolve(false);
             }, waitMs);
             timer.unref?.();
-            this.#waiting.add(granted);
+            this.#waiting.set(granted, lane);
         });
     }
 
-    /** Hand the slot to the longest waiter, or give it back. */
-    #release(): void {
-        const next = this.#waiting.values().next().value;
-        if (next) {
-            this.#waiting.delete(next);
-            next();
+    /**
+     * Give the slot back, then hand one to the longest waiter it can go to: a
+     * schedule waiter at a full schedule share doesn't hold up a delivery.
+     */
+    #release(lane: InvokeLane): void {
+        this.#active--;
+        if (lane === 'schedule') this.#scheduleActive--;
+        for (const [granted, waiting] of this.#waiting) {
+            if (!this.#free(waiting)) continue;
+            this.#waiting.delete(granted);
+            this.#take(waiting);
+            granted();
             return;
         }
-        this.#active--;
     }
 
     /** One pool per process, built on first use and closed with the server. */
@@ -446,6 +784,65 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     }
 }
 
+/** What a failure the dispatcher marked as its own reads as. */
+const dispatchAnswer = (
+    reason: string,
+    status: number,
+): EventsInvokeAnswer => ({
+    status: null,
+    error: `dispatcher: ${reason} (${status})`,
+    dispatchReason: reason,
+});
+
+/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
+const timeoutError = (): Error =>
+    new DOMException(
+        'The operation was aborted due to timeout',
+        'TimeoutError',
+    );
+
+/** One line of a batch answer, or null when it is not one. */
+export const parseAnswerLine = (
+    line: string,
+):
+    | { i: number; started: true }
+    | { i: number; answer: EventsInvokeAnswer }
+    | null => {
+    let raw: unknown;
+    try {
+        raw = JSON.parse(line);
+    } catch {
+        return null;
+    }
+    if (!raw || typeof raw !== 'object') return null;
+    const { i, started, status, handled, error, retryAfter, dispatch } =
+        raw as Record<string, unknown>;
+    if (!Number.isInteger(i) || (i as number) < 0) return null;
+    if (started === true) return { i: i as number, started: true };
+    if (typeof status !== 'number') return null;
+    if (typeof dispatch === 'string')
+        return { i: i as number, answer: dispatchAnswer(dispatch, status) };
+    return {
+        i: i as number,
+        answer: answerFrom({
+            status,
+            headers: {
+                get: (name) => {
+                    if (name === EVENTS_HANDLED_HEADER)
+                        return handled === true ? '1' : null;
+                    if (name === EVENTS_ERROR_HEADER)
+                        return typeof error === 'string' ? error : null;
+                    if (name === 'retry-after')
+                        return typeof retryAfter === 'string'
+                            ? retryAfter
+                            : null;
+                    return null;
+                },
+            },
+        }),
+    };
+};
+
 /** Whether deploying the script is what fixes this answer. */
 const selfDeployable = (result: EventsInvokeAnswer): boolean =>
     result.status === null

src/backend/controllers/auth/AuthController.test.ts

@@ -5132,6 +5132,26 @@ describe('AuthController phone verification — staging & reuse', () => {
         expect(staged).toBe('+14155550123');
     });
 
+    it('makes an email-gate-last account verify its card before its phone', async () => {
+        const { user, actor } = await makeUserAndActor({
+            requires_phone_verification: 1,
+            requires_card_verification: 1,
+        });
+        await server.stores.user.updateMetadata(user.id, {
+            emailGateLast: true,
+        });
+        const prelude = stubPrelude();
+        await withClients({ prelude }, async () => {
+            await expect(
+                controller.handleSendConfirmPhone(
+                    makeReq({ phone: '+14155550123' }, { actor }),
+                    makeRes(),
+                ),
+            ).rejects.toMatchObject({ statusCode: 409 });
+        });
+        expect(prelude.createVerification).not.toHaveBeenCalled();
+    });
+
     it('forwards an abuse-extension veto as 429 with the opaque reason, and sends nothing', async () => {
         const { user, actor } = await makeUserAndActor();
         const emitAndWait = vi.fn(
@@ -5197,7 +5217,7 @@ describe('AuthController phone verification — staging & reuse', () => {
         );
     });
 
-    it('carries the client dispatch id on the sent signal', async () => {
+    it('carries the client dispatch id on the check and sent events', async () => {
         const { actor } = await makeUserAndActor();
         const emit = vi.fn();
         const emitAndWait = vi.fn(async () => {});
@@ -5218,6 +5238,11 @@ describe('AuthController phone verification — staging & reuse', () => {
             expect.objectContaining({ dispatch_id: 'disp-123' }),
             expect.anything(),
         );
+        expect(emitAndWait).toHaveBeenCalledWith(
+            'puter.phone-verification.check',
+            expect.objectContaining({ dispatch_id: 'disp-123' }),
+            expect.anything(),
+        );
     });
 
     it('does NOT emit the sent signal when the send fails upstream', async () => {

src/backend/controllers/auth/AuthController.ts

@@ -66,6 +66,7 @@ import {
     isCardFallbackEnabled,
     phoneAttemptsKey,
 } from '../../util/cardFallback.js';
+import { isEmailGateLast } from '../../util/verificationOrder.js';
 import { sessionCookieFlags } from '../../util/cookieFlags.js';
 import { cleanEmail, isStorableEmail } from '../../util/email.js';
 import { isGodmodeApp } from '../../util/godmodeApps.js';
@@ -1220,6 +1221,18 @@ export class AuthController extends PuterController {
         }
     }
 
+    /** The email-gate-last confirmation email; a failure never fails the gate. */
+    private async sendDeferredConfirmation(userId: number): Promise<void> {
+        try {
+            await this.services.signup.sendDeferredConfirmationEmail(userId);
+        } catch (e) {
+            console.warn(
+                '[verification] deferred confirmation email failed:',
+                e,
+            );
+        }
+    }
+
     private async isCardFallbackEligible(user: {
         id: number;
         requires_phone_verification?: boolean | number | null;
@@ -1265,6 +1278,12 @@ export class AuthController extends PuterController {
             throw new HttpError(403, 'Account suspended.', {
                 legacyCode: 'account_suspended',
             });
+        if (isEmailGateLast(user) && user.requires_card_verification)
+            throw new HttpError(
+                409,
+                'Card verification must be completed first.',
+                { legacyCode: 'conflict' },
+            );
         if (!this.clients.prelude?.isConfigured())
             throw await this.smsSendError(
                 503,
@@ -1335,6 +1354,7 @@ export class AuthController extends PuterController {
             user_uid: user.uuid,
             phone: parsed.e164,
             device_fingerprint: req.deviceFingerprint ?? null,
+            dispatch_id: dispatchId ?? null,
             allowed: true,
             reason: null as string | null,
             card_fallback_instead: false,
@@ -1573,6 +1593,7 @@ export class AuthController extends PuterController {
             requires_phone_verification: 0,
             phone: pendingPhone,
         });
+        await this.sendDeferredConfirmation(user.id);
 
         // Run the verified-event listeners synchronously (emitAndWait, not
         // fire-and-forget emit) so a carrier-based card-verification waiver in
@@ -1653,7 +1674,11 @@ export class AuthController extends PuterController {
             res.json({ card_verified: true });
             return;
         }
-        if (user.requires_phone_verification && !fallbackEligible)
+        if (
+            user.requires_phone_verification &&
+            !fallbackEligible &&
+            !isEmailGateLast(user)
+        )
             throw new HttpError(
                 409,
                 'Phone verification must be completed first.',
@@ -1716,6 +1741,7 @@ export class AuthController extends PuterController {
             await this.stores.user.update(user.id, {
                 requires_card_verification: 0,
             });
+            await this.sendDeferredConfirmation(user.id);
             res.json({ card_verified: true, disabled: true });
             return;
         }
@@ -1775,7 +1801,11 @@ export class AuthController extends PuterController {
             res.json({ card_verified: true });
             return;
         }
-        if (user.requires_phone_verification && !fallbackEligible)
+        if (
+            user.requires_phone_verification &&
+            !fallbackEligible &&
+            !isEmailGateLast(user)
+        )
             throw new HttpError(
                 409,
                 'Phone verification must be completed first.',
@@ -1816,6 +1846,7 @@ export class AuthController extends PuterController {
             await this.stores.user.update(user.id, {
                 requires_card_verification: 0,
             });
+            await this.sendDeferredConfirmation(user.id);
             res.json({ card_verified: true, disabled: true });
             return;
         }
@@ -1840,6 +1871,7 @@ export class AuthController extends PuterController {
             requires_card_verification: 0,
             ...(clearedPhoneGate ? { requires_phone_verification: 0 } : {}),
         });
+        await this.sendDeferredConfirmation(user.id);
 
         // Awaited like `user.phone-verified`, so whatever a listener grants on
         // verification is in place before the client refreshes. emitAndWait

src/backend/controllers/broadcast/BroadcastController.ts

@@ -19,6 +19,7 @@
 
 import type { Request, Response } from 'express';
 import { Controller, Post } from '../../core/http/decorators.js';
+import { schedulesUnavailable } from '../../core/schedules.js';
 import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
 import type { ForwardBatch } from '../../services/events/forwardQueue.js';
 import { PuterController } from '../types.js';
@@ -108,6 +109,42 @@ export class BroadcastController extends PuterController {
         );
         res.status(200).json({ ok: true, ...reply });
     }
+
+    /**
+     * Schedule changes a region made for schedules this one owns, handed to
+     * `services.schedules`; `events_disabled` on a server without it.
+     */
+    @Post('/events/schedules', { subdomain: '*' })
+    async schedules(req: Request, res: Response): Promise<void> {
+        const broadcast = this.services.broadcast as unknown as
+            BroadcastService | undefined;
+        if (!broadcast) {
+            res.status(503).json({
+                error: { message: 'Broadcast service not registered' },
+            });
+            return;
+        }
+
+        const verified = await broadcast.verifySignedRequest(
+            req.rawBody,
+            broadcastHeaders(req),
+        );
+        if (!verified.ok) {
+            res.status(verified.status ?? 400).json({
+                error: { message: verified.message ?? 'Bad request' },
+            });
+            return;
+        }
+        if (verified.info?.ignored) {
+            res.status(200).json({ ok: true, ...verified.info });
+            return;
+        }
+
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        const reply = await schedules.receivePeerChanges(req.body);
+        res.status(200).json({ ok: true, ...reply });
+    }
 }
 
 const broadcastHeaders = (req: Request) => {

src/backend/controllers/drivers/DriverController.test.ts

@@ -736,6 +736,42 @@ describe('DriverController driver-method lifecycle events', () => {
             rejectReason: 'blocked in test',
         });
     });
+
+    it('answers 429 when a before listener vetoes with rejectStatus 429', async () => {
+        const actor = await makeUserActor();
+        await grantKvPerm(actor);
+        const key = `throttle-${uuidv4()}`;
+        server.clients.event.on(
+            'driver.puter-kvstore.set.before',
+            (_k, data) => {
+                if ((data.args as { key?: string })?.key === key) {
+                    data.allow = false;
+                    data.rejectStatus = 429;
+                }
+            },
+        );
+
+        const req = makeReq(
+            {
+                interface: 'puter-kvstore',
+                method: 'set',
+                args: { key, value: 'v' },
+            },
+            actor,
+        );
+        await expect(
+            runWithContext({ actor }, () =>
+                routes['POST /call'](
+                    req,
+                    makeRes() as unknown as Response,
+                    () => {},
+                ),
+            ),
+        ).rejects.toMatchObject({
+            statusCode: 429,
+            legacyCode: 'too_many_requests',
+        });
+    });
 });
 
 describe('DriverController.#handleListInterfaces', () => {

src/backend/controllers/drivers/DriverController.ts

@@ -31,6 +31,7 @@ import {
     checkDriverRateLimit,
 } from '../../core/http/middleware/rateLimit.js';
 import type { PuterRouter } from '../../core/http/PuterRouter.js';
+import type { LifecycleRejectStatus } from '../../clients/event/types.js';
 import type { DriverMeta } from '../../drivers/meta.js';
 import {
     isCreditExhaustion,
@@ -334,7 +335,8 @@ export class DriverController extends PuterController {
 
         if (req.actor) {
             const permService = this.services.permission as unknown as
-                PermissionService | undefined;
+                | PermissionService
+                | undefined;
             if (permService) {
                 // Build via PermissionUtil.join so any `:` in a driver or
                 // interface name is escaped — raw interpolation would let a
@@ -475,6 +477,7 @@ export class DriverController extends PuterController {
             args,
             allow: true as boolean,
             rejectReason: undefined as string | undefined,
+            rejectStatus: undefined as LifecycleRejectStatus | undefined,
         };
         await this.clients.event?.emitAndWait(
             `driver.${ifaceName}.${method}.before`,
@@ -496,11 +499,12 @@ export class DriverController extends PuterController {
                 },
                 {},
             );
+            const throttled = beforeEvent.rejectStatus === 429;
             throw new HttpError(
-                403,
+                throttled ? 429 : 403,
                 beforeEvent.rejectReason ??
                     `Blocked by policy: ${ifaceName}:${method}`,
-                { legacyCode: 'forbidden' },
+                { legacyCode: throttled ? 'too_many_requests' : 'forbidden' },
             );
         }
 

src/backend/controllers/events/EventsController.ts

@@ -25,6 +25,11 @@ import { bodyRecord } from '../../core/http/requestBody.js';
 import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
 import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
 import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    schedulesUnavailable,
+    type ScheduleServiceContract,
+} from '../../core/schedules.js';
 import { normalizeLimit } from '../../util/pagination.js';
 import { PuterController } from '../types.js';
 import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -169,6 +174,88 @@ export class EventsController extends PuterController {
         res.json({});
     }
 
+    // -- Schedules ---------------------------------------------------
+    //
+    // An app's handler run on a timer, served by `services.schedules` when a
+    // server has it and answered `events_disabled` when it doesn't. Creating
+    // one needs an app context; listing, reading and removing are scoped the
+    // way the durable verbs are. The create and remove verbs spend the
+    // subscribe and unsubscribe budgets in the service, shared with those
+    // verbs.
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /** POST /events/schedules — create a schedule for the calling app. */
+    @Post('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async createSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
+    }
+
+    /** GET /events/schedules — the caller's schedules, one page at a time. */
+    @Get('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async listSchedules(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const query = (req.query ?? {}) as Record<string, unknown>;
+
+        const page = await this.#schedules().list(actor, {
+            limit: normalizeLimit(query.limit, {
+                cap: SCHEDULE_LIST_LIMIT_CAP,
+            }),
+            cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
+            includeTotal: query.includeTotal === 'true',
+        });
+
+        res.json({
+            items: page.items,
+            ...(page.cursor ? { cursor: page.cursor } : {}),
+            ...(page.total !== undefined ? { total: page.total } : {}),
+        });
+    }
+
+    /**
+     * POST /events/schedules/remove — end one schedule. An id the caller does
+     * not hold reads as absent.
+     */
+    @Post('/schedules/remove', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async removeSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
+        res.json({});
+    }
+
+    /**
+     * GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
+     * caller does not hold reads as absent.
+     */
+    @Get('/schedules/:scheduleId', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async getSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().get(actor, req.params.scheduleId));
+    }
+
     // -- Cross-user key-value handles --------------------------------
 
     /**
@@ -425,10 +512,14 @@ export class EventsController extends PuterController {
             res.json({ deployed: true });
             return;
         }
-        // `failed` and `throttled` are worth another attempt: the rest say
-        // this script is not what the app's handlers currently are.
+        // `failed`, `throttled` and `busy` are worth another attempt: the rest
+        // say this script is not what the app's handlers currently are.
         res.status(
-            outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
+            outcome === 'failed' ||
+                outcome === 'throttled' ||
+                outcome === 'busy'
+                ? 502
+                : 404,
         ).json({
             deployed: false,
             reason: outcome,
@@ -455,8 +546,8 @@ export class EventsController extends PuterController {
             );
             return;
         }
-        this.services.events.useWorkerMissHandler((appUid, script) =>
-            deployer.ensure(appUid, script),
+        this.services.events.useWorkerMissHandler((appUid, script, options) =>
+            deployer.ensure(appUid, script, options),
         );
     }
 

src/backend/controllers/events/limits.test.ts

@@ -41,16 +41,30 @@ import {
     EVENTS_SINGLE_DELIVERY_LIMIT,
     EVENTS_WORKER_INVOCATION_LIMIT,
     limitFor,
+    tiered,
     type TieredLimit,
 } from './limits.js';
 
 const tiers: Array<[string, TieredLimit]> = [
-    ['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
+    [
+        'durable subscriptions per account',
+        EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    ],
     ['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
     ['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
     ['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
 ];
 
+describe('tiered', () => {
+    it('gives paid plans the base, and the free and temporary tiers their own', () => {
+        const limit = tiered(30, 20, 10);
+        expect(limitFor(limit, null)).toBe(30);
+        expect(limitFor(limit, 'some_plan')).toBe(30);
+        expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
+        expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
+    });
+});
+
 describe('the tiered subscription quotas', () => {
     it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
         for (const n of Object.values(tier.bySubscription))
@@ -105,9 +119,9 @@ describe('the tiered subscription quotas', () => {
     });
 
     it('holds an unrecognised plan to the paid base', () => {
-        expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
-            EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
-        );
+        expect(
+            limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
+        ).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
         expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
             EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
         );
@@ -131,9 +145,9 @@ describe('the delivery budgets', () => {
             expect(
                 limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
             ).toBe(matched);
-            expect(
-                limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
-            ).toBe(evaluated);
+            expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
+                evaluated,
+            );
             expect(evaluated).toBeGreaterThanOrEqual(matched);
         }
     });

src/backend/controllers/events/limits.ts

@@ -53,7 +53,12 @@ export interface TieredLimit {
     bySubscription: Record<string, number>;
 }
 
-const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
+/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
+export const tiered = (
+    paid: number,
+    free: number,
+    temp: number,
+): TieredLimit => ({
     limit: paid,
     bySubscription: {
         [DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -448,11 +453,13 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
 // finish backs off the same way but is not the handler's failure, so it never
 // counts toward that run.
 
-/** Wait before the first retry of a delivery a handler could not take. */
-export const EVENTS_RETRY_BASE_MS = 2_000;
-
-/** Longest wait between retries, however many have failed. */
-export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+// Wait before the first retry of a delivery a handler could not take, the
+// longest wait, and the doubling between them. Shared with system jobs.
+export {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from '../../util/retryBackoff.js';
 
 /**
  * Handler failures in a row before a subscription is suspended. Counted per
@@ -468,13 +475,6 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
  */
 export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
 
-/** How long to hold a delivery whose handler has failed `attempts` times. */
-export const deliveryBackoffMs = (attempts: number): number =>
-    Math.min(
-        EVENTS_RETRY_MAX_MS,
-        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
-    );
-
 // -- Suspended backlog -----------------------------------------------
 //
 // A suspended subscription stops metering but keeps what it is owed, and that

src/backend/controllers/events/scheduleRoutes.test.ts

@@ -0,0 +1,274 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The schedule routes are thin delegates: without `services.schedules` every
+ * one answers 503 `events_disabled`; with it, each hands its request over and
+ * answers what it returns.
+ */
+
+import type { Request, Response } from 'express';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    ScheduleServiceContract,
+    ScheduleView,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { BroadcastController } from '../broadcast/BroadcastController.js';
+import type { EventsController } from './EventsController.js';
+
+let server: PuterServer;
+let events: EventsController;
+let broadcast: BroadcastController;
+let actor: Actor;
+let saved: unknown;
+
+const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
+
+const view = {
+    scheduleId: SCHEDULE_ID,
+    kind: 'recurring',
+    appUid: 'app-x',
+} as ScheduleView;
+
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const fakeSchedules = () =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(async () => view),
+        list: vi.fn(async () => ({
+            items: [view],
+            cursor: 'next',
+            total: 3,
+        })),
+        get: vi.fn(async () => view),
+        remove: vi.fn(async () => {}),
+        receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    }) satisfies ScheduleServiceContract;
+
+const req = (init: {
+    body?: unknown;
+    query?: Record<string, unknown>;
+    params?: Record<string, string>;
+}): Request =>
+    ({
+        actor,
+        body: init.body ?? {},
+        query: init.query ?? {},
+        params: init.params ?? {},
+        headers: {},
+        rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
+    }) as unknown as Request;
+
+const res = () => {
+    const sent: { status: number; body: unknown } = {
+        status: 200,
+        body: undefined,
+    };
+    const response = {
+        json: vi.fn((body: unknown) => {
+            sent.body = body;
+            return response;
+        }),
+        status: vi.fn((code: number) => {
+            sent.status = code;
+            return response;
+        }),
+        setHeader: vi.fn(() => response),
+    };
+    return { response: response as unknown as Response, sent };
+};
+
+/** Each route, called as the router would. */
+const routes: Array<[string, (response: Response) => Promise<void>]> = [
+    [
+        'POST /events/schedules',
+        (r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
+    ],
+    [
+        'GET /events/schedules',
+        (r) => events.listSchedules(req({ query: { limit: '5' } }), r),
+    ],
+    [
+        'GET /events/schedules/:scheduleId',
+        (r) =>
+            events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
+    ],
+    [
+        'POST /events/schedules/remove',
+        (r) =>
+            events.removeSchedule(
+                req({ body: { scheduleId: SCHEDULE_ID } }),
+                r,
+            ),
+    ],
+    [
+        'POST /broadcast/events/schedules',
+        (r) =>
+            broadcast.schedules(
+                req({ body: { namespace: 'n', changes: [] } }),
+                r,
+            ),
+    ],
+];
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    events = server.controllers.events as unknown as EventsController;
+    broadcast = server.controllers.broadcast as unknown as BroadcastController;
+    actor = makeActor({
+        user: { id: 1, uuid: 'u-1', username: 'someone' },
+    } as never);
+    saved = slot().schedules;
+    vi.spyOn(
+        server.services.broadcast,
+        'verifySignedRequest',
+    ).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+});
+
+afterAll(async () => {
+    vi.restoreAllMocks();
+    await server?.shutdown();
+});
+
+describe('without services.schedules', () => {
+    it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
+        setSchedules(undefined);
+        const { response } = res();
+        let thrown: unknown;
+        try {
+            await call(response);
+        } catch (err) {
+            thrown = err;
+        }
+        expect(isHttpError(thrown)).toBe(true);
+        expect(thrown).toMatchObject({
+            statusCode: 503,
+            legacyCode: 'events_disabled',
+            // The designed answer, so it never pages.
+            noAlarm: true,
+        });
+    });
+
+    it('refuses an unsigned peer batch before saying schedules are off', async () => {
+        setSchedules(undefined);
+        vi.spyOn(
+            server.services.broadcast,
+            'verifySignedRequest',
+        ).mockResolvedValueOnce({
+            ok: false,
+            status: 401,
+            message: 'bad signature',
+        } as never);
+        const { response, sent } = res();
+        await broadcast.schedules(
+            req({ body: { namespace: 'n', changes: [] } }),
+            response,
+        );
+        expect(sent.status).toBe(401);
+    });
+});
+
+describe('with services.schedules', () => {
+    it('creates through createFor and answers its view', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.createSchedule(req({ body: { every: 60_000 } }), response);
+        expect(schedules.createFor).toHaveBeenCalledWith(actor, {
+            every: 60_000,
+        });
+        expect(sent.body).toEqual(view);
+    });
+
+    it('lists with the limit capped, the cursor and includeTotal', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.listSchedules(
+            req({
+                query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
+            }),
+            response,
+        );
+        expect(schedules.list).toHaveBeenCalledWith(actor, {
+            limit: 200,
+            cursor: 'c',
+            includeTotal: true,
+        });
+        expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
+    });
+
+    it('reads one through get', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.getSchedule(
+            req({ params: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual(view);
+    });
+
+    it('removes through remove and answers {}', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.removeSchedule(
+            req({ body: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual({});
+    });
+
+    it('hands a signed peer batch to receivePeerChanges', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        const body = { namespace: 'n', changes: [] };
+        await broadcast.schedules(req({ body }), response);
+        expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
+        expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
+    });
+});

src/backend/controllers/events/workerDeploy.test.ts

@@ -25,8 +25,13 @@
 
 import MockRedis from 'ioredis-mock';
 import { describe, expect, it } from 'vitest';
+import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
 import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
-import { EventsWorkerDeployer } from './workerDeploy.js';
+import {
+    EventsWorkerDeployer,
+    LocalEventsInvokeTransport,
+    type EventsDeployOutcome,
+} from './workerDeploy.js';
 import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
 import { handlerSetHash } from '../../services/events/workerSource.js';
 import type { IConfig } from '../../types.js';
@@ -176,3 +181,151 @@ describe('EventsWorkerDeployer.ensure', () => {
         expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
     });
 });
+
+/** A server's schedules with a deploy gate that records and can refuse. */
+const withGate = (
+    layers: Layers,
+    refuse: (capped: boolean) => boolean = () => false,
+) => {
+    const gated: Array<{ script: string; capped: boolean }> = [];
+    const gate = {
+        gateDeploy: async (
+            script: string,
+            options: { capped: boolean },
+            deploy: () => Promise<EventsDeployOutcome>,
+        ): Promise<EventsDeployOutcome> => {
+            gated.push({ script, capped: options.capped });
+            return refuse(options.capped) ? 'busy' : deploy();
+        },
+    };
+    const deployer = new EventsWorkerDeployer({
+        ...layers,
+        services: { ...layers.services, schedules: gate },
+    } as unknown as Layers);
+    return { deployer, gated };
+};
+
+describe('the schedules` deploy gate', () => {
+    it('is skipped without schedules, or with schedules that have no gate', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        await expect(
+            new EventsWorkerDeployer(layers).ensure('app-x', SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        const gateless = new EventsWorkerDeployer({
+            ...layers,
+            services: { ...layers.services, schedules: {} },
+        } as unknown as Layers);
+        await expect(
+            gateless.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('takes every deploy, capped only for a schedule run', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers);
+
+        await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
+            'deployed',
+        );
+        await expect(
+            deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('deployed');
+        expect(gated).toEqual([
+            { script: SCRIPT, capped: false },
+            { script: SCRIPT, capped: true },
+        ]);
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('answers a refused schedule run busy, and a delivery still deploys', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer } = withGate(layers, (capped) => capped);
+
+        await expect(
+            deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('busy');
+        await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
+            'deployed',
+        );
+        expect(created).toEqual([SCRIPT]);
+    });
+
+    it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const scheduled = deployer.ensure('app-x', SCRIPT, {
+            lane: 'schedule',
+        });
+        const deliveries = [
+            deployer.ensure('app-x', SCRIPT),
+            deployer.ensure('app-x', SCRIPT),
+        ];
+        await expect(scheduled).resolves.toBe('busy');
+        await expect(Promise.all(deliveries)).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(created).toEqual([SCRIPT]);
+        expect(gated.map((g) => g.capped)).toEqual([true, false]);
+    });
+
+    it('lets a schedule run join a delivery`s deploy in flight', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const delivery = deployer.ensure('app-x', SCRIPT);
+        const scheduled = deployer.ensure('app-x', SCRIPT, {
+            lane: 'schedule',
+        });
+        await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
+        expect(created).toEqual([SCRIPT]);
+    });
+});
+
+describe('LocalEventsInvokeTransport', () => {
+    it('renders each request as it goes out, after a cold deploy', async () => {
+        const bodies: string[] = [];
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async (request: Request) => {
+                        bodies.push(await request.text());
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async () => {
+                    resident = true;
+                    return 'deployed';
+                },
+            } as unknown as EventsWorkerDeployer,
+        );
+        let renders = 0;
+        const call: EventsInvokeCall = {
+            script: SCRIPT,
+            appUid: 'app-x',
+            key: 'k',
+            body: '{"n":0}',
+            timeoutMs: 5000,
+            render: () => `{"n":${++renders}}`,
+        };
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
+    });
+});

src/backend/controllers/events/workerDeploy.ts

@@ -29,6 +29,7 @@ import {
 import type {
     EventsInvokeAnswer,
     EventsInvokeCall,
+    InvokeLane,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import {
     EVENTS_INVOKE_PATH,
@@ -36,6 +37,7 @@ import {
     failedAnswer,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import type { puterServices } from '../../services/index.js';
+import type { IExtensionServiceInstances } from '../../services/types.js';
 import {
     eventsInvokeKey,
     eventsWorkerScope,
@@ -61,13 +63,25 @@ import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
 
 /** Why a deploy could not happen, for the callers that answer differently. */
 export type EventsDeployOutcome =
-    'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
+    | 'deployed'
+    | 'stale'
+    | 'no-handlers'
+    | 'no-owner'
+    | 'throttled'
+    | 'failed'
+    | 'busy';
+
+/** Whose call asked for the deploy: a schedule run's is held to a share. */
+export interface EventsDeployOptions {
+    lane?: InvokeLane;
+}
 
 interface DeployLayers {
     config: IConfig;
     clients: LayerInstances<typeof puterClients>;
     stores: LayerInstances<typeof puterStores>;
-    services: LayerInstances<typeof puterServices>;
+    services: LayerInstances<typeof puterServices> &
+        Pick<IExtensionServiceInstances, 'schedules'>;
     drivers: LayerInstances<typeof puterDrivers>;
 }
 
@@ -77,6 +91,12 @@ interface DeployLayers {
  */
 const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
 
+interface InFlightDeploy {
+    /** A schedule run's, which the region's deploy slots may refuse. */
+    capped: boolean;
+    promise: Promise<EventsDeployOutcome>;
+}
+
 interface DeployResult {
     success?: boolean;
     errors?: unknown[];
@@ -85,7 +105,7 @@ interface DeployResult {
 export class EventsWorkerDeployer {
     readonly #layers: DeployLayers;
     /** Collapses the burst of misses one cold app produces into one deploy. */
-    readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
+    readonly #inFlight = new Map<string, InFlightDeploy>();
 
     constructor(layers: DeployLayers) {
         this.#layers = layers;
@@ -104,15 +124,79 @@ export class EventsWorkerDeployer {
      * A request for anything else is `stale` — the set moved on, and the next
      * delivery resolves the new script rather than this one being revived.
      */
-    ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
+    ensure(
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions = {},
+    ): Promise<EventsDeployOutcome> {
         const key = `${appUid}|${script}`;
+        const capped = options.lane === 'schedule';
         const existing = this.#inFlight.get(key);
-        if (existing) return existing;
-        const started = this.#deploy(appUid, script).finally(() =>
-            this.#inFlight.delete(key),
+        if (!existing) return this.#start(key, appUid, script, options);
+        if (capped || !existing.capped) return existing.promise;
+        return this.#afterCapped(
+            existing.promise,
+            key,
+            appUid,
+            script,
+            options,
+        );
+    }
+
+    #start(
+        key: string,
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const entry: InFlightDeploy = {
+            capped: options.lane === 'schedule',
+            promise: this.#gated(appUid, script, options).finally(() => {
+                if (this.#inFlight.get(key) === entry)
+                    this.#inFlight.delete(key);
+            }),
+        };
+        this.#inFlight.set(key, entry);
+        return entry.promise;
+    }
+
+    /**
+     * A delivery that joined a schedule run's deploy: `busy` means no deploy
+     * started, and a delivery is never refused a slot, so it asks again.
+     */
+    async #afterCapped(
+        capped: Promise<EventsDeployOutcome>,
+        key: string,
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const outcome = await capped;
+        if (outcome !== 'busy') return outcome;
+        const current = this.#inFlight.get(key);
+        if (current && !current.capped) return current.promise;
+        return this.#start(key, appUid, script, options);
+    }
+
+    /**
+     * Every deploy goes through the schedules' deploy gate when a server has
+     * one: a per-script lock, so a burst of runs of an evicted script deploys
+     * it once, and a region-wide cap that a schedule run's deploy (`capped`)
+     * may be refused with `busy`. A delivery's deploy is never refused.
+     */
+    #gated(
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const deploy = () => this.#deploy(appUid, script);
+        return (
+            this.#layers.services.schedules?.gateDeploy?.(
+                script,
+                { capped: options.lane === 'schedule' },
+                deploy,
+            ) ?? deploy()
         );
-        this.#inFlight.set(key, started);
-        return started;
     }
 
     async #deploy(
@@ -299,6 +383,8 @@ export class LocalEventsInvokeTransport {
     }
 
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        // Built per attempt, so a send time stamped into the body is taken
+        // after a cold deploy rather than before it.
         const request = (): Request =>
             new Request(
                 `http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -308,7 +394,7 @@ export class LocalEventsInvokeTransport {
                         'content-type': 'application/json',
                         'x-puter-events-key': call.key,
                     },
-                    body: call.body,
+                    body: call.render ? call.render() : call.body,
                 },
             );
 
@@ -319,9 +405,14 @@ export class LocalEventsInvokeTransport {
             const outcome = await this.#deployer.ensure(
                 call.appUid,
                 call.script,
+                { lane: call.lane },
             );
             if (outcome !== 'deployed')
-                return { status: null, error: `deploy: ${outcome}` };
+                return {
+                    status: null,
+                    error: `deploy: ${outcome}`,
+                    ...(outcome === 'busy' ? { deployBusy: true } : {}),
+                };
 
             const retried = await this.#dispatch(request(), call.timeoutMs);
             if (retried === null)

src/backend/core/http/middleware/gates.test.ts

@@ -836,6 +836,34 @@ describe('requireVerifiedAccount', () => {
         expectHttpError(got, 403, 'card_verification_required');
     });
 
+    it('asks an email-gate-last account for card, then phone, then email', () => {
+        const pending = {
+            uuid: 'u-1',
+            requires_email_confirmation: true,
+            email_confirmed: false,
+            requires_phone_verification: true,
+            requires_card_verification: true,
+            metadata: { emailGateLast: true },
+        };
+        const gateFor = (user: Record<string, unknown>) =>
+            runGate(requireVerifiedAccount(), { actor: { user } });
+        expectHttpError(gateFor(pending), 403, 'card_verification_required');
+        expectHttpError(
+            gateFor({ ...pending, requires_card_verification: false }),
+            403,
+            'phone_verification_required',
+        );
+        expectHttpError(
+            gateFor({
+                ...pending,
+                requires_card_verification: false,
+                requires_phone_verification: false,
+            }),
+            403,
+            'email_confirmation_required',
+        );
+    });
+
     it('returns 403 two_factor_required for a seat whose team requires 2FA', async () => {
         const teams = { getOrgSeat: async () => ({ require_2fa: 1 }) };
         const got = await runGateAsync(requireTeam2fa(teams), {

src/backend/core/http/middleware/gates.ts

@@ -32,6 +32,7 @@ import {
     type CardFallbackDeps,
 } from '../../../util/cardFallback';
 import { reserveUsernames } from '../../../util/reservedUsernames';
+import { isEmailGateLast } from '../../../util/verificationOrder';
 import {
     isAccountContext,
     isAppActor,
@@ -309,11 +310,25 @@ export const requireVerifiedAccount = (): RequestHandler =>
 export const assertVerifiedAccount = (
     user: AccountGateUser | undefined,
 ): void => {
-    if (user?.requires_email_confirmation && !user?.email_confirmed) {
-        throw new HttpError(403, 'Please confirm your email to continue', {
-            legacyCode: 'email_confirmation_required',
-        });
-    }
+    const assertEmail = () => {
+        if (user?.requires_email_confirmation && !user?.email_confirmed) {
+            throw new HttpError(403, 'Please confirm your email to continue', {
+                legacyCode: 'email_confirmation_required',
+            });
+        }
+    };
+    const assertCard = () => {
+        if (user?.requires_card_verification) {
+            throw new HttpError(403, 'Please verify your card to continue', {
+                legacyCode: 'card_verification_required',
+            });
+        }
+    };
+    // Email → phone → card, or card → phone → email for an email-gate-last
+    // account, so the client is sent to the gate it has to clear next.
+    const emailLast = isEmailGateLast(user);
+    if (emailLast) assertCard();
+    else assertEmail();
     if (user?.requires_phone_verification) {
         throw new HttpError(
             403,
@@ -323,11 +338,8 @@ export const assertVerifiedAccount = (
             },
         );
     }
-    if (user?.requires_card_verification) {
-        throw new HttpError(403, 'Please verify your card to continue', {
-            legacyCode: 'card_verification_required',
-        });
-    }
+    if (emailLast) assertEmail();
+    else assertCard();
     // A team seat signs in on a password its administrator still holds,
     // so it reaches nothing until it has replaced that password.
     if (user?.requires_password_change) {

src/backend/core/http/routeLifecycle.test.ts

@@ -290,6 +290,26 @@ describe('createRouteLifecycleMiddleware', () => {
         expect(after).toHaveLength(0);
     });
 
+    it('answers 429 when the vetoing listener asks for it', async () => {
+        const events = makeEvents();
+        events.on(`${BASE}.before` as never, (_k, data) => {
+            (data as RouteLifecycleEvent).allow = false;
+            (data as RouteLifecycleEvent).rejectReason = 'slow down';
+            (data as RouteLifecycleEvent).rejectStatus = 429;
+        });
+        const reject = record(events, `${BASE}.reject` as keyof EventMap);
+
+        const mw = createRouteLifecycleMiddleware(events, POST, PATH);
+        const res = new FakeRes();
+        await mw(makeReq('u-9'), res as unknown as Response, () => {});
+
+        expect(res.statusCode).toBe(429);
+        expect(res.body).toMatchObject({
+            error: { code: 'too_many_requests', message: 'slow down' },
+        });
+        expect(reject[0]).toMatchObject({ statusCode: 429 });
+    });
+
     it('treats a listener-sent response as a terminal after (not a reject) with the real status', async () => {
         const events = makeEvents();
         events.on(`${BASE}.before` as never, (_k, data) => {

src/backend/core/http/routeLifecycle.ts

@@ -20,6 +20,7 @@
 import type { Request, RequestHandler } from 'express';
 import { actorUid } from '../actor';
 import type { EventClient } from '../../clients/event/EventClient';
+import type { LifecycleRejectStatus } from '../../clients/event/types';
 import type { RouteMethod } from './types';
 
 type RoutePathValue = string | RegExp | Array<string | RegExp> | undefined;
@@ -100,30 +101,33 @@ export const createRouteLifecycleMiddleware = (
             ...base,
             allow: true as boolean,
             rejectReason: undefined as string | undefined,
+            rejectStatus: undefined as LifecycleRejectStatus | undefined,
         };
         if (observeBefore) {
             await events.emitAndWait(beforeKey, beforeEvent, {});
         }
 
         // Explicit veto: a listener set `allow = false`. Emit `reject` and
-        // answer 403 — unless the listener already wrote its own response, in
-        // which case we report that status rather than a misleading 403.
+        // answer 403 (or the listener's 429) — unless the listener already
+        // wrote its own response, in which case we report that status.
         if (beforeEvent.allow === false) {
+            const status = beforeEvent.rejectStatus === 429 ? 429 : 403;
             events.emit(
                 `${keyBase}.reject`,
                 {
                     phase: 'reject',
                     ...base,
-                    statusCode: res.headersSent ? res.statusCode : 403,
+                    statusCode: res.headersSent ? res.statusCode : status,
                     durationMs: Date.now() - startedAt,
                     rejectReason: beforeEvent.rejectReason,
                 },
                 {},
             );
             if (!res.headersSent) {
-                res.status(403).json({
+                res.status(status).json({
                     error: {
-                        code: 'forbidden',
+                        code:
+                            status === 429 ? 'too_many_requests' : 'forbidden',
                         message:
                             beforeEvent.rejectReason ?? 'Blocked by policy',
                     },

src/backend/core/http/types.ts

@@ -76,6 +76,7 @@ export type AccountGateUser = Partial<
         | 'phone'
         | 'card_fingerprint'
         | 'otp_enabled'
+        | 'metadata'
     >
 > & { id?: number };
 

src/backend/core/schedules.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { isHttpError } from './http/HttpError.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    SCHEDULE_PEER_PATH,
+    schedulesUnavailable,
+} from './schedules.js';
+
+describe('schedules contract', () => {
+    it('answers an absent schedules service with a 503 events_disabled', () => {
+        const err = schedulesUnavailable();
+        expect(isHttpError(err)).toBe(true);
+        expect(err.statusCode).toBe(503);
+        expect(err.legacyCode).toBe('events_disabled');
+        // Expected on a server without schedules, so it must not page.
+        expect(err.noAlarm).toBe(true);
+    });
+
+    it('keeps the list cap and the peer path the extension relies on', () => {
+        expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
+        expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
+    });
+});

src/backend/core/schedules.ts

@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * What core asks of schedules. Nothing here is an implementation: an extension
+ * registers `services.schedules`, and a server without one answers every
+ * schedule route and `notify({ sendAt })` with `events_disabled`. Called by
+ * extensions; keep these signatures stable.
+ */
+
+import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
+import type { AppMessage } from '../services/notification/appMessage.js';
+import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
+import type { DurableSubscription } from '../stores/events/types.js';
+import type { PageResult } from '../util/pagination.js';
+import type { Actor } from './actor.js';
+import { HttpError } from './http/HttpError.js';
+
+/** Why a schedule ended. */
+export type ScheduleEndReason =
+    | 'removed'
+    | 'ended'
+    | 'app_deleted'
+    | 'uninstalled'
+    | 'permission_revoked'
+    | 'holder_gone'
+    | 'app_gone'
+    | 'orphaned'
+    | 'delivered'
+    | 'dropped';
+
+/** What happened to a set of subscription rows schedules own. */
+export type ScheduleSubscriptionChange =
+    | {
+          type: 'suspended';
+          reason: SuspendedReason;
+          /** For a reason that never lifts; defaults to that reason. */
+          endReason?: ScheduleEndReason;
+      }
+    | { type: 'resumed' }
+    | { type: 'ended'; endReason: ScheduleEndReason };
+
+/** A schedule that keeps running, one that runs once, or a delayed notification. */
+export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
+
+/**
+ * A schedule as its holder sees it. Times are epoch ms. `context` values are
+ * never returned, as for a persistent subscription: only its keys and a hash.
+ */
+export interface ScheduleView {
+    scheduleId: string;
+    kind: ScheduleViewKind;
+    appUid: string;
+    /** `null` once ended: the handler binding goes with the schedule. */
+    handlerName: string | null;
+    contextKeys: string[] | null;
+    contextHash: string | null;
+    /** Interval of an `every` schedule. */
+    every: number | null;
+    cron: string | null;
+    tz: string;
+    /** First allowed run; a one-shot's only run. */
+    startAt: number;
+    /** Last allowed run, inclusive. */
+    endsAt: number | null;
+    /** `sending` once a delayed send has started; it can't be cancelled then. */
+    state: 'active' | 'suspended' | 'sending' | 'ended';
+    suspendedReason: string | null;
+    endReason: string | null;
+    endedAt: number | null;
+    /**
+     * The slot it runs for next; `null` while suspended or sending, and once
+     * ended.
+     */
+    nextRunAt: number | null;
+    createdAt: number;
+}
+
+export interface ScheduleListRequest {
+    limit: number;
+    cursor?: string;
+    includeTotal: boolean;
+}
+
+/** The caps a holder's schedules are held to, from their plan. */
+export interface ScheduleQuota {
+    recurringPerUser: number;
+    recurringPerApp: number;
+    oneShotsPerUser: number;
+    oneShotsPerApp: number;
+    /** Worst-case runs per hour × 1000, summed over recurring rows. */
+    budgetPerUser: number;
+    budgetPerApp: number;
+}
+
+/** A caller cleared to supply `sendAt`, and when it asked for. */
+export interface DeferredSendPlan {
+    sendAt: number;
+    /** Due within 5 seconds, or past: sent now rather than stored. */
+    immediate: boolean;
+    holderUserId: number;
+    appUid: string;
+    handlerDepth: number;
+    quota: ScheduleQuota;
+}
+
+/** What a delayed app message stores until it is sent. */
+export interface DeferredAppMessage {
+    message: AppMessage;
+    /** A KV share handle naming the recipient. */
+    to?: string;
+}
+
+/**
+ * A condition on one row of another table that statements embed as `EXISTS
+ * (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
+ */
+export interface SqlGuard {
+    table: string;
+    where: string;
+    values: unknown[];
+}
+
+/** Ceiling of a schedule listing page. */
+export const SCHEDULE_LIST_LIMIT_CAP = 200;
+
+/** The signed peer route a region's schedule changes are sent to. */
+export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
+
+export const schedulesUnavailable = (): HttpError =>
+    new HttpError(503, 'Schedules are not available on this server', {
+        legacyCode: 'events_disabled',
+        // The designed answer of a server without schedules, not a fault.
+        noAlarm: true,
+    });
+
+export interface ScheduleServiceContract {
+    /**
+     * Keep schedules in step with a change to the subscription rows they own.
+     * Never throws: the subscription change has already happened.
+     */
+    onSubscriptionsChanged(
+        rows: readonly DurableSubscription[],
+        change: ScheduleSubscriptionChange,
+    ): Promise<void>;
+    createFor(
+        actor: Actor,
+        body: Record<string, unknown>,
+    ): Promise<ScheduleView>;
+    list(
+        actor: Actor,
+        request: ScheduleListRequest,
+    ): Promise<PageResult<ScheduleView>>;
+    get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
+    remove(actor: Actor, scheduleId: unknown): Promise<void>;
+    receivePeerChanges(body: unknown): Promise<{ applied: number }>;
+    /** Checks a caller supplying `sendAt`; `events_disabled` when off. */
+    prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
+    createDeferredAppMessage(
+        plan: DeferredSendPlan,
+        message: DeferredAppMessage,
+    ): Promise<{ scheduleId: string; sendAt: number }>;
+    /**
+     * Runs an events worker deploy under the schedules' deploy gate; `capped`
+     * for a schedule run's. A passthrough when schedules are off.
+     */
+    gateDeploy?(
+        script: string,
+        options: { capped: boolean },
+        deploy: () => Promise<EventsDeployOutcome>,
+    ): Promise<EventsDeployOutcome>;
+}

src/backend/drivers/notification/NotificationDriver.sendAt.test.ts

@@ -0,0 +1,493 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `notify({ sendAt })` without and with `services.schedules`, and the deferred
+ * mailbox write an extension runs inside its own transaction, fenced on a guard
+ * in a table of its own (a test table here).
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    DeferredSendPlan,
+    ScheduleServiceContract,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import type {
+    DeferredWriteTx,
+    NotificationDriver,
+} from './NotificationDriver.js';
+
+vi.setConfig({ testTimeout: 120_000 });
+
+const BOOT_TIMEOUT_MS = 480_000;
+const TEST_TABLE = 'deferred_send_test';
+
+let server: PuterServer;
+let holder: Actor;
+let app: { id: number; uid: string };
+let saved: unknown;
+let nextId = 1;
+
+const db = () => server.clients.db;
+const driver = () =>
+    server.drivers.notifications as unknown as NotificationDriver;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+const asApp = (): Actor => makeActor({ user: holder.user, app });
+
+const notify = (fields: Record<string, unknown>): Promise<unknown> =>
+    runWithContext({ actor: asApp() }, () =>
+        driver().create({
+            object: { value: { type: 'app.message', title: 'Hi', ...fields } },
+        }),
+    );
+
+const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
+    try {
+        await run;
+    } catch (err) {
+        if (isHttpError(err)) return err.legacyCode;
+        throw err;
+    }
+    return undefined;
+};
+
+const mailbox = () =>
+    db().read(
+        'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
+            'ORDER BY `id`',
+        [holder.user!.id, app.uid],
+    );
+
+/** A row's value: JSON text on sqlite, parsed on the other engines. */
+const valueOf = (row: Record<string, unknown>) =>
+    typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
+
+/** A send claimed under `token`, as an extension's own row would be. */
+const claimed = async (token: string): Promise<number> => {
+    const id = nextId++;
+    await db().write(
+        `INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
+        [id, token],
+    );
+    return id;
+};
+
+/** The transaction an extension wraps the write in: its row, then the mailbox. */
+const txFor = (id: number, token: string): DeferredWriteTx => ({
+    guard: {
+        table: TEST_TABLE,
+        where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
+        values: [id, token],
+    },
+    async commit(entries: readonly BatchEntry[]) {
+        await db().batchWrite([
+            {
+                statement:
+                    `UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
+                    "WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
+                values: [id, token],
+            },
+            ...entries,
+        ]);
+        const [row] = await db().pread(
+            `SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
+            [id],
+        );
+        return row?.state === 'delivered' && row.token === token;
+    },
+});
+
+const write = (
+    sendKey: string,
+    tx: DeferredWriteTx,
+    over: {
+        payload?: string | null;
+        appUid?: string;
+        signal?: AbortSignal;
+    } = {},
+) =>
+    driver().deferredAppMessages.write(
+        {
+            holderUserId: holder.user!.id,
+            appUid: over.appUid ?? app.uid,
+            payload:
+                over.payload === undefined
+                    ? JSON.stringify({ message: { title: 'Later' } })
+                    : over.payload,
+            sendKey,
+            signal: over.signal ?? new AbortController().signal,
+        },
+        tx,
+    );
+
+const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
+            sendAt: 0,
+            immediate: false,
+            holderUserId: holder.user!.id,
+            appUid: app.uid,
+            handlerDepth: 0,
+            quota: {
+                recurringPerUser: 1,
+                recurringPerApp: 1,
+                oneShotsPerUser: 1,
+                oneShotsPerApp: 1,
+                budgetPerUser: 1,
+                budgetPerApp: 1,
+            },
+            ...plan,
+        })),
+        createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
+            scheduleId: 'sched-1',
+            sendAt: p.sendAt,
+        })),
+    }) satisfies ScheduleServiceContract;
+
+beforeAll(async () => {
+    server = await setupTestServer({
+        events: { enabled: true },
+    } as IConfig);
+    const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+        requires_email_confirmation: false,
+    });
+    const row = (await server.stores.user.getById(user.id))!;
+    holder = makeActor({
+        user: {
+            id: row.id,
+            uuid: row.uuid,
+            username: row.username,
+            email: row.email ?? null,
+            email_confirmed: true,
+        } as Actor['user'],
+    });
+    const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
+    app = (await server.stores.app.create(
+        { name, title: name, index_url: `https://${name}.test/` },
+        { ownerUserId: row.id },
+    )) as { id: number; uid: string };
+    await server.services.permission.grantUserAppPermission(
+        holder,
+        app.uid,
+        EVENTS_BACKGROUND_PERMISSION,
+    );
+    await db().write(
+        `CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
+            '`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
+        [],
+    );
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
+        app.uid,
+    ]);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('notify({ sendAt }) without services.schedules', () => {
+    it('is refused events_disabled, even when due now, storing nothing', async () => {
+        setSchedules(undefined);
+        expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
+            'events_disabled',
+        );
+        expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
+            'events_disabled',
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends as before without sendAt', async () => {
+        setSchedules(undefined);
+        const sent = (await notify({ text: 'now' })) as { uid: string };
+        expect(sent.uid).toBeTruthy();
+        expect(await mailbox()).toHaveLength(1);
+    });
+});
+
+describe('notify({ sendAt }) with services.schedules', () => {
+    it('hands a later send to createDeferredAppMessage and answers its id', async () => {
+        const at = Date.now() + 3_600_000;
+        const schedules = fakeSchedules({ sendAt: at });
+        setSchedules(schedules);
+
+        await expect(notify({ sendAt: at, text: 'later' })).resolves.toEqual({
+            scheduled: { scheduleId: 'sched-1', sendAt: at },
+        });
+        expect(schedules.prepareSendAt).toHaveBeenCalledWith(
+            expect.objectContaining({ user: holder.user }),
+            at,
+        );
+        expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
+            expect.objectContaining({ sendAt: at }),
+            {
+                message: expect.objectContaining({
+                    title: 'Hi',
+                    text: 'later',
+                }),
+            },
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends at once when the plan says it is due', async () => {
+        const schedules = fakeSchedules({ immediate: true });
+        setSchedules(schedules);
+
+        const sent = (await notify({ sendAt: Date.now() })) as {
+            uid: string;
+        };
+        expect(sent.uid).toBeTruthy();
+        expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
+        expect(await mailbox()).toHaveLength(1);
+    });
+
+    it('passes a refusal from prepareSendAt through', async () => {
+        const schedules = fakeSchedules({});
+        schedules.prepareSendAt.mockRejectedValueOnce(
+            Object.assign(new Error('scoped'), {}),
+        );
+        setSchedules(schedules);
+        await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+describe('deferredAppMessages.write', () => {
+    it('writes one row keyed by the send, in the caller`s transaction', async () => {
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(write(key, txFor(id, token))).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: key, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('writes nothing when the guard no longer holds: another claim took it', async () => {
+        const id = await claimed(uuidv4());
+        const key = uuidv4();
+
+        await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('replaces the open row of its group, which takes the key', async () => {
+        const open = (await notify({ group: 'g', text: 'first' })) as {
+            uid: string;
+        };
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(
+            write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g' },
+                }),
+            }),
+        ).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('drops a payload it can`t read, and a sender whose app is gone', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        await expect(
+            write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
+        ).resolves.toBe('dropped');
+        await expect(
+            write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
+        ).resolves.toBe('dropped');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('writes nothing once the caller`s claim is lost', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        const lost = new AbortController();
+        lost.abort();
+
+        await expect(
+            write(uuidv4(), txFor(id, token), { signal: lost.signal }),
+        ).rejects.toThrow();
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+/**
+ * A second claim taking a send over while the first one's transaction is open.
+ * Needs two writers at once with row locks; sqlite serialises writers, so this
+ * runs on postgres only.
+ */
+describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
+    'a deferred write racing another writer',
+    () => {
+        /**
+         * Hold the first statement matching `match` on its connection, before
+         * or after it runs, until released.
+         */
+        const holdStatement = (match: RegExp, when: 'before' | 'after') => {
+            const client = db() as unknown as {
+                query: (
+                    target: unknown,
+                    sql: string,
+                    params?: unknown[],
+                ) => Promise<unknown>;
+            };
+            const original = client.query.bind(client);
+            let reached!: () => void;
+            let release!: () => void;
+            const atBarrier = new Promise<void>((r) => (reached = r));
+            const released = new Promise<void>((r) => (release = r));
+            let held = false;
+            vi.spyOn(client as never, 'query' as never).mockImplementation(
+                (async (target: unknown, sql: string, params?: unknown[]) => {
+                    if (held || !match.test(sql))
+                        return original(target, sql, params);
+                    held = true;
+                    if (when === 'before') {
+                        reached();
+                        await released;
+                        return original(target, sql, params);
+                    }
+                    const result = await original(target, sql, params);
+                    reached();
+                    await released;
+                    return result;
+                }) as never,
+            );
+            return { atBarrier, release };
+        };
+
+        const DELIVERED = /SET `state` = 'delivered'/;
+        const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
+
+        /** Whether `promise` is still pending after `ms`. */
+        const stillPending = async (promise: Promise<unknown>, ms: number) => {
+            let settled = false;
+            const watched = (async () => {
+                try {
+                    await promise;
+                } finally {
+                    settled = true;
+                }
+            })();
+            await new Promise((r) => setTimeout(r, ms));
+            void watched;
+            return !settled;
+        };
+
+        it('a transaction that opened before a takeover writes nothing', async () => {
+            const stale = uuidv4();
+            const id = await claimed(stale);
+            const key = uuidv4();
+            const gate = holdStatement(DELIVERED, 'before');
+            const first = write(key, txFor(id, stale));
+            await gate.atBarrier;
+
+            await db().write(
+                `UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
+                [uuidv4(), id],
+            );
+            gate.release();
+            await expect(first).resolves.toBe('lost');
+            expect(
+                await server.stores.notification.getBySendKey(key),
+            ).toBeNull();
+            expect(await mailbox()).toEqual([]);
+        });
+
+        it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
+            const open = (await notify({ group: 'g', text: 'first' })) as {
+                uid: string;
+            };
+            const token = uuidv4();
+            const id = await claimed(token);
+            const key = uuidv4();
+            const gate = holdStatement(REPLACE, 'after');
+            const deferred = write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g', text: 'later' },
+                }),
+            });
+            await gate.atBarrier;
+
+            const immediate = notify({ group: 'g', text: 'newer' });
+            // Past the group lock's wait, then blocked on the row.
+            expect(await stillPending(immediate, 2_500)).toBe(true);
+            gate.release();
+            await expect(deferred).resolves.toBe('delivered');
+            await immediate;
+
+            const rows = await mailbox();
+            expect(rows).toHaveLength(1);
+            expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
+            expect(valueOf(rows[0]).text).toBe('newer');
+        });
+    },
+);

src/backend/drivers/notification/NotificationDriver.ts

@@ -51,6 +51,14 @@ import {
     resolveNotificationWrite,
     type NotificationTypeName,
 } from '../../services/notification/notificationTypes.js';
+import {
+    schedulesUnavailable,
+    type DeferredAppMessage,
+    type DeferredSendPlan,
+    type ScheduleServiceContract,
+    type SqlGuard,
+} from '../../core/schedules.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { withRedisLock } from '../../util/redisLock.js';
 import { PuterDriver } from '../types.js';
 import type { Actor } from '../../core/actor.js';
@@ -67,6 +75,44 @@ const GROUP_LOCK_RETRY_MS = 25;
 
 type AppMessageWindow = 'minute' | 'day';
 
+export type { SqlGuard };
+
+/** What a delayed app message's fire-time write needs from its row. */
+export interface DeferredWriteInput {
+    holderUserId: number;
+    appUid: string;
+    /** The stored {@link DeferredAppMessage}, as JSON text. */
+    payload: string | null;
+    /** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
+    sendKey: string;
+    /** Aborted when the caller's claim is lost. */
+    signal: AbortSignal;
+}
+
+/**
+ * The caller's transaction the mailbox write commits in. Atomicity is the
+ * caller's to keep; the type can't enforce it.
+ */
+export interface DeferredWriteTx {
+    /**
+     * What every mailbox statement is fenced on: the caller's row marked
+     * delivered by this claim.
+     */
+    guard: SqlGuard;
+    /**
+     * Must be the whole delivery in one step: the caller's guarded transition
+     * of its row to delivered and `entries`, in that order, in one
+     * `batchWrite`, then a read of the row on the primary. True only when that
+     * read shows this claim holding the row as delivered. Splitting the
+     * transition from `entries` lets one slot write two rows.
+     */
+    commit(entries: readonly BatchEntry[]): Promise<boolean>;
+}
+
+/** How a delayed send's write stood when it stopped. */
+export type DeferredWriteOutcome =
+    'delivered' | 'dropped' | 'lost' | 'senderSuspended';
+
 /** The mailbox slice a call named, in the terms the query takes it. */
 interface MailboxScope {
     audiences: readonly string[];
@@ -124,6 +170,21 @@ const parseNotifyTarget = (to: unknown): string => {
     return target.handle;
 };
 
+/** A stored delayed message, or `null` for one that can't be read back. */
+const parseDelayedAppMessage = (
+    payload: string | null,
+): DeferredAppMessage | null => {
+    try {
+        const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
+        if (typeof stored?.message?.title !== 'string') return null;
+        if (stored.to !== undefined && typeof stored.to !== 'string')
+            return null;
+        return stored;
+    } catch {
+        return null;
+    }
+};
+
 /** Whether a row belongs to the slice, as the same SQL scope selected it. */
 const inScope = (
     row: Record<string, unknown>,
@@ -354,7 +415,7 @@ export class NotificationDriver extends PuterDriver {
         object: Record<string, unknown>,
         value: Record<string, unknown>,
     ): Promise<unknown> {
-        const { type, to, ...fields } = value;
+        const { type, to, sendAt, ...fields } = value;
         if (type !== APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
         }
@@ -370,6 +431,12 @@ export class NotificationDriver extends PuterDriver {
         const handle = to === undefined ? undefined : parseNotifyTarget(to);
         const message = parseAppMessage(fields);
         assertValueFits(value);
+        const timing =
+            sendAt === undefined || sendAt === null
+                ? null
+                : await this.#schedules().prepareSendAt(actor, sendAt);
+        if (timing && !timing.immediate)
+            return this.#scheduleAppMessage(actor, message, handle, timing);
 
         if (handle === undefined) {
             const uid = await this.#sendAppMessage(
@@ -410,12 +477,23 @@ export class NotificationDriver extends PuterDriver {
         appUid: string,
         message: AppMessage,
     ): Promise<string> {
-        const send = () =>
-            this.#replaceOrWriteAppMessage(recipientId, appUid, message);
+        return this.#oneSendPerGroup(recipientId, appUid, message, () =>
+            this.#replaceOrWriteAppMessage(recipientId, appUid, message),
+        );
+    }
+
+    /**
+     * One send per group at a time, so a burst into an empty group writes one
+     * row that the rest replace. A lock that is busy past the wait or
+     * unreachable still sends: the race costs a second row and its day slot.
+     */
+    #oneSendPerGroup<T>(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+        send: () => Promise<T>,
+    ): Promise<T> {
         if (!message.group) return send();
-        // One send per group at a time, so a burst into an empty group writes
-        // one row that the rest replace. A lock that is busy past the wait or
-        // unreachable still sends: the race costs a second row and its day slot.
         return withRedisLock(
             this.clients.redis,
             `notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
@@ -430,6 +508,163 @@ export class NotificationDriver extends PuterDriver {
         );
     }
 
+    // -- Delayed sends -----------------------------------------------
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /**
+     * Store an app's message to go out at `sendAt`. A recipient named through a
+     * share is checked now without spending its budget; every limit is spent
+     * when it goes, and the share is checked again then.
+     */
+    async #scheduleAppMessage(
+        actor: UserActor,
+        message: AppMessage,
+        handle: string | undefined,
+        plan: DeferredSendPlan,
+    ): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
+        if (handle !== undefined)
+            await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
+                spendBudget: false,
+            });
+        return {
+            scheduled: await this.#schedules().createDeferredAppMessage(
+                plan,
+                handle === undefined ? { message } : { message, to: handle },
+            ),
+        };
+    }
+
+    /**
+     * For extensions. A field, not a method, so `/drivers/call` can't reach it.
+     */
+    readonly deferredAppMessages = {
+        write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
+            this.#writeDeferredAppMessage(input, tx),
+    };
+
+    /**
+     * Check the sender and recipient as they stand now, spend the recipient's
+     * limits, and write the mailbox row through `tx`, fenced on its guard.
+     * Refusals are the same `HttpError`s an immediate send throws.
+     */
+    async #writeDeferredAppMessage(
+        input: DeferredWriteInput,
+        tx: DeferredWriteTx,
+    ): Promise<DeferredWriteOutcome> {
+        const { holderUserId, sendKey, signal } = input;
+        const sender = await this.services.events.resolveScheduleActor(
+            holderUserId,
+            input.appUid,
+        );
+        if (!sender) {
+            // A lagging replica can miss an app the primary has: retried.
+            const there =
+                (await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
+                (await this.stores.user.getById(holderUserId)) !== null;
+            if (there) throw new Error('the sender could not be resolved');
+            return 'dropped';
+        }
+        if (sender.user?.suspended) return 'senderSuspended';
+        // Uninstalled since, and the cancel at uninstall missed it.
+        const appId = sender.effectiveApp?.id;
+        if (
+            typeof appId !== 'number' ||
+            !(await this.stores.permission.hasAnyUserAppPerm(
+                holderUserId,
+                appId,
+            ))
+        )
+            return 'dropped';
+        const stored = parseDelayedAppMessage(input.payload);
+        if (!stored) return 'dropped';
+        const { message } = stored;
+
+        let recipientId = holderUserId;
+        let appUid = input.appUid;
+        if (stored.to !== undefined) {
+            const recipient =
+                await this.services.kvShare.resolveNotifyRecipient(
+                    sender,
+                    stored.to,
+                );
+            recipientId = recipient.recipientUserId;
+            appUid = recipient.appUid;
+        }
+
+        return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
+            // Primary, as for an immediate send.
+            const open = message.group
+                ? await this.stores.notification.getUnseenByGroup(
+                      recipientId,
+                      appUid,
+                      message.group,
+                      { primary: true },
+                  )
+                : null;
+            await this.#spendAppMessageLimits(
+                recipientId,
+                appUid,
+                open ? ['minute'] : ['minute', 'day'],
+            );
+            message.app = appMessageSender(
+                appUid,
+                await this.stores.app.getByUid(appUid),
+            );
+            // Another attempt may hold the lease now; hand the claim back.
+            if (signal.aborted) throw new Error('delayed send lost its lease');
+
+            const registered = resolveNotificationWrite(
+                APP_MESSAGE_TYPE,
+                appUid,
+            );
+            const delivered = await tx.commit(
+                this.stores.notification.sendEntries(
+                    {
+                        sendKey,
+                        userId: recipientId,
+                        value: { ...message, type: registered.type },
+                        type: registered.type,
+                        audience: registered.audience,
+                        appUid,
+                        group: message.group ?? null,
+                        replaceUid: open ? String(open.uid) : null,
+                        at: Date.now(),
+                    },
+                    tx.guard,
+                ),
+            );
+            if (!delivered) return 'lost';
+            if (open)
+                await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
+            return 'delivered';
+        });
+    }
+
+    /**
+     * A replacement that found its row shown since the lookup wrote a new one,
+     * which costs a day slot. Already written, so a full day is not a refusal.
+     */
+    async #spendDayIfFresh(
+        recipientId: number,
+        appUid: string,
+        open: Record<string, unknown>,
+        sendKey: string,
+    ): Promise<void> {
+        try {
+            const written =
+                await this.stores.notification.getBySendKey(sendKey);
+            if (written && written.uid !== open.uid)
+                await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
+        } catch {
+            // The row is written; a slot not spent is the only loss.
+        }
+    }
+
     async #replaceOrWriteAppMessage(
         recipientId: number,
         appUid: string,

src/backend/package.json

@@ -57,7 +57,7 @@
     "openai": "^6.34.0",
     "otpauth": "^9.5.2",
     "pg": "^8.21.0",
-    "postal-mime": "^3.0.1",
+    "postal-mime": "^4.2.0",
     "replicate": "^1.0.0",
     "sharp": "^0.35.5",
     "smtp-server": "^3.19.24",

src/backend/server.test.ts

@@ -36,6 +36,10 @@ import { Controller, Get } from './core/http/decorators.ts';
 import { HttpError } from './core/http/HttpError.ts';
 import { extension, extensionStore } from './extensions.ts';
 import { PuterServer } from './server.ts';
+import {
+    JobScheduler,
+    type JobDefinition,
+} from './services/jobs/JobScheduler.ts';
 import {
     allocateEphemeralPort,
     createTestUser,
@@ -1539,6 +1543,69 @@ describe('PuterServer lifecycle hooks', () => {
     });
 });
 
+describe('PuterServer shutdown during prepare', () => {
+    it('waits for prepare hooks the drain already started before tearing down', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        const order: string[] = [];
+        (server.services as unknown as Record<string, unknown>).slowPrepare = {
+            onServerPrepareShutdown: async () => {
+                order.push('prepare:start');
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                order.push('prepare:done');
+            },
+            onServerShutdown: () => {
+                order.push('shutdown');
+            },
+        };
+
+        await server.prepareShutdown({ drainMs: 0 });
+        // The listener's close callback starts the prepare hooks.
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await server.shutdown();
+
+        expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
+    });
+});
+
+/**
+ * An extension may take over `services.jobs` (a fleet-wide scheduler). Core
+ * services register their jobs on it at start, so it must keep the built-in's
+ * first place in the layer.
+ */
+describe('PuterServer jobs replacement', () => {
+    it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
+        const registered: string[] = [];
+        class ReplacementJobs extends JobScheduler {
+            override register(name: string, definition: JobDefinition) {
+                registered.push(name);
+                super.register(name, definition);
+            }
+        }
+        const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
+        extension.registerService('jobs', ReplacementJobs as never);
+        let server: PuterServer | undefined;
+        try {
+            server = await setupTestServer();
+            const { jobs } = server.services;
+            expect(jobs).toBeInstanceOf(ReplacementJobs);
+            expect(Object.keys(server.services)[0]).toBe('jobs');
+            expect(extension.import('service').jobs).toBeInstanceOf(
+                ReplacementJobs,
+            );
+            // Only the replacement gets lifecycle hooks.
+            expect(started.mock.contexts).toEqual([jobs]);
+            expect(registered).toContain('auth.sessions.purge');
+        } finally {
+            delete (extensionStore.services as Record<string, unknown>).jobs;
+            started.mockRestore();
+            await server?.shutdown();
+        }
+    });
+});
+
 /**
  * Each layer's `onServerShutdown` has to run while the layers beneath it (the
  * ones it writes through) are still up, so teardown goes top-down: drivers,

src/backend/server.ts

@@ -1765,16 +1765,19 @@ export class PuterServer {
         }
     }
 
-    #prepareShutdownHooksRan = false;
+    #prepareShutdownHooks: Promise<void> | null = null;
 
     /**
      * Run every layer's `onServerPrepareShutdown` exactly once, whichever of
-     * `prepareShutdown()` / `shutdown()` gets there first.
+     * `prepareShutdown()` / `shutdown()` gets there first; the other waits for
+     * that run to finish.
      */
-    async #runPrepareShutdownHooks() {
-        if (this.#prepareShutdownHooksRan) return;
-        this.#prepareShutdownHooksRan = true;
-        await this.#runHook('bottomUp', 'onServerPrepareShutdown');
+    #runPrepareShutdownHooks(): Promise<void> {
+        this.#prepareShutdownHooks ??= this.#runHook(
+            'bottomUp',
+            'onServerPrepareShutdown',
+        );
+        return this.#prepareShutdownHooks;
     }
 
     async prepareShutdown(info: ServerDrainInfo = {}) {

src/backend/services/auth/AuthService.test.ts

@@ -24,6 +24,7 @@ import { makeActor, type Actor } from '../../core/actor.js';
 import { PuterServer } from '../../server.js';
 import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
 import { setupTestServer } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import { generateDefaultFsentries } from '../../util/userProvisioning.js';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS } from '../permission/consts.js';
@@ -4048,7 +4049,6 @@ describe('AuthService (integration)', () => {
     });
 
     describe('purgeEndedSessions', () => {
-        const GATE_KEY = 'sessions:purge:gate';
         const DAY = 24 * 60 * 60;
 
         const makeSession = async (
@@ -4077,8 +4077,15 @@ describe('AuthService (integration)', () => {
             return rows.length > 0;
         };
 
+        const idOf = async (uuid: string) => {
+            const [row] = await server.clients.db.read(
+                'SELECT `id` FROM `sessions` WHERE `uuid` = ?',
+                [uuid],
+            );
+            return Number(row.id);
+        };
+
         it('deletes rows ended past their retention and keeps the rest', async () => {
-            await server.clients.redis.del(GATE_KEY);
             const user = await makeUser();
             const id = user.id;
             const oldRevoked = await makeSession(id, 'asset', 'revoked_at', 31);
@@ -4100,7 +4107,7 @@ describe('AuthService (integration)', () => {
             const live = await makeSession(id, 'web');
 
             expect(
-                await authService.purgeEndedSessions(),
+                await authService.purgeEndedSessions(testJobContext()),
             ).toBeGreaterThanOrEqual(3);
 
             expect(await exists(oldRevoked)).toBe(false);
@@ -4112,34 +4119,84 @@ describe('AuthService (integration)', () => {
             expect(await exists(live)).toBe(true);
         });
 
-        it('skips while another run holds the gate', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            await server.clients.redis.set(GATE_KEY, 'other-node');
+        it('starts from the saved cursor and clears it once the walk is done', async () => {
             const user = await makeUser();
-            const oldRevoked = await makeSession(
-                user.id,
-                'asset',
-                'revoked_at',
-                31,
-            );
+            const below = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const above = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const ctx = testJobContext({ afterId: await idOf(below) });
 
-            expect(await authService.purgeEndedSessions()).toBeNull();
-            expect(await exists(oldRevoked)).toBe(true);
-            await server.clients.redis.del(GATE_KEY);
+            expect(
+                await authService.purgeEndedSessions(ctx),
+            ).toBeGreaterThanOrEqual(1);
+            expect(await exists(below)).toBe(true);
+            expect(await exists(above)).toBe(false);
+            expect(ctx.saved).toEqual([null]);
         });
 
-        it('frees the gate when the walk fails', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            const spy = vi
-                .spyOn(server.clients.db, 'read')
-                .mockRejectedValueOnce(new Error('db down'));
+        it('clears its own checkpoints once a walk completes, so the next run starts at zero', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 11,
+            );
+            const ranges: number[] = [];
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    return 0;
+                });
             try {
-                await expect(authService.purgeEndedSessions()).rejects.toThrow(
-                    'db down',
+                const first = testJobContext();
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(11);
+                // Checkpointed at ten ranges, then cleared at the end.
+                expect(first.saved).toEqual([{ afterId: SPAN * 10 }, null]);
+
+                ranges.length = 0;
+                await authService.purgeEndedSessions(
+                    testJobContext(first.saved.at(-1)),
                 );
-                expect(await server.clients.redis.get(GATE_KEY)).toBeNull();
+                expect(ranges[0]).toBe(0);
+                expect(ranges).toHaveLength(11);
             } finally {
-                spy.mockRestore();
+                purge.mockRestore();
+                vi.restoreAllMocks();
+            }
+        });
+
+        it('stops between ranges once aborted, and the next run resumes at the saved cursor', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 30,
+            );
+            const ranges: number[] = [];
+            const first = testJobContext();
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    // Aborted mid-walk, as a lost lease or a shutdown would.
+                    if (ranges.length === 12) first.abort();
+                    return 0;
+                });
+            try {
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(12);
+                // Saved every 10 ranges, and where it stopped.
+                expect(first.saved).toEqual([
+                    { afterId: SPAN * 10 },
+                    { afterId: SPAN * 12 },
+                ]);
+
+                ranges.length = 0;
+                const next = testJobContext(first.saved.at(-1));
+                await authService.purgeEndedSessions(next);
+                expect(ranges[0]).toBe(SPAN * 12);
+                expect(ranges.at(-1)).toBe(SPAN * 29);
+                expect(next.saved.at(-1)).toBeNull();
+            } finally {
+                purge.mockRestore();
+                vi.restoreAllMocks();
             }
         });
     });

src/backend/services/auth/AuthService.ts

@@ -46,6 +46,7 @@ import type { puterServices } from '../index';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
 import type { GrantMeta } from '../permission/PermissionService.js';
+import { savedAfterId, type JobRunContext } from '../jobs/JobScheduler.js';
 import { PuterService } from '../types';
 import type {
     AccessTokenPayload,
@@ -64,13 +65,14 @@ const MAX_ORIGIN_UID_GENERATIONS = 8;
 // rows double as sign-in history, so they stay longer.
 const SESSION_RETENTION_SECONDS = 30 * 24 * 60 * 60;
 const WEB_SESSION_RETENTION_SECONDS = 365 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per week.
-const SESSION_PURGE_CHECK_MS = 60 * 60 * 1000;
-const SESSION_PURGE_GATE_KEY = 'sessions:purge:gate';
-const SESSION_PURGE_GATE_SECONDS = 7 * 24 * 60 * 60;
+const SESSION_PURGE_JOB = 'auth.sessions.purge';
+const SESSION_PURGE_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
+const SESSION_PURGE_LEASE_MS = 30 * 60 * 1000;
 // Ids per DELETE, with a pause between, so replicas keep up.
 const SESSION_PURGE_SPAN = 5000;
 const SESSION_PURGE_PAUSE_MS = 50;
+/** Ranges walked between saves of the cursor. */
+const SESSION_PURGE_SAVE_EVERY = 10;
 
 const nowSeconds = (): number => Math.floor(Date.now() / 1000);
 
@@ -138,11 +140,17 @@ export interface AuthResult {
 export class AuthService extends PuterService {
     declare protected services: LayerInstances<typeof puterServices>;
 
-    #sessionPurge: ReturnType<typeof setInterval> | null = null;
-    #stopping = false;
-
     override onServerStart(): void {
-        this.#armSessionPurge();
+        this.services.jobs.register(SESSION_PURGE_JOB, {
+            every: SESSION_PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: SESSION_PURGE_LEASE_MS,
+            run: async (ctx) => {
+                const purged = await this.purgeEndedSessions(ctx);
+                console.log(`[auth] purged ${purged} ended sessions`);
+            },
+        });
 
         // Users implicitly hold read access to their own email — needed for
         // any permission-gated path that asks for `user:<uuid>:email:read`
@@ -164,85 +172,54 @@ export class AuthService extends PuterService {
         });
     }
 
-    override onServerPrepareShutdown(): void {
-        this.#stopping = true;
-        if (this.#sessionPurge) clearInterval(this.#sessionPurge);
-        this.#sessionPurge = null;
-    }
-
     // -- Session purge -----------------------------------------------
 
     /**
      * Delete sessions revoked or expired longer ago than their retention,
-     * walking the table by id. Returns the rows deleted, or null when another
-     * node already holds this week's gate.
+     * walking the table by id from the cursor an interrupted run saved. Returns
+     * the rows deleted; stops early once `signal` aborts.
      */
-    async purgeEndedSessions(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            SESSION_PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            SESSION_PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeEndedSessions({
+        signal,
+        state,
+        saveState,
+    }: JobRunContext): Promise<number> {
         const now = nowSeconds();
         const cutoffs = {
             cutoff: now - SESSION_RETENTION_SECONDS,
             webCutoff: now - WEB_SESSION_RETENTION_SECONDS,
         };
+        const maxId = await this.stores.session.maxId();
+        let afterId = savedAfterId(state);
         let purged = 0;
-        let finished = false;
-        try {
-            const maxId = await this.stores.session.maxId();
-            for (
-                let afterId = 0;
-                afterId < maxId;
-                afterId += SESSION_PURGE_SPAN
-            ) {
-                if (this.#stopping) return purged;
-                purged += await this.stores.session.purgeEndedInRange(
-                    afterId,
-                    SESSION_PURGE_SPAN,
-                    cutoffs,
-                );
-                await new Promise((resolve) =>
-                    setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A walk cut short frees the gate, so another node restarts it on
-            // its next check instead of next week.
-            if (!finished) {
+        let unsaved = 0;
+        while (afterId < maxId) {
+            if (signal.aborted) {
+                // Kept for the next run if this one still holds the job.
                 try {
-                    await this.clients.redis.del(SESSION_PURGE_GATE_KEY);
+                    await saveState({ afterId });
                 } catch {
-                    // The gate expires on its own.
+                    // Lost the job; the next run redoes these ranges.
                 }
+                return purged;
             }
-        }
-    }
-
-    #armSessionPurge(): void {
-        const timer = setInterval(() => {
-            void this.#runSessionPurge();
-        }, SESSION_PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#sessionPurge = timer;
-    }
-
-    async #runSessionPurge(): Promise<void> {
-        try {
-            const purged = await this.purgeEndedSessions();
-            if (purged !== null) {
-                console.log(`[auth] purged ${purged} ended sessions`);
+            purged += await this.stores.session.purgeEndedInRange(
+                afterId,
+                SESSION_PURGE_SPAN,
+                cutoffs,
+            );
+            afterId += SESSION_PURGE_SPAN;
+            if (++unsaved >= SESSION_PURGE_SAVE_EVERY) {
+                await saveState({ afterId });
+                unsaved = 0;
             }
-        } catch (err) {
-            console.warn('[auth] session purge failed', err);
+            await new Promise((resolve) =>
+                setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
+            );
         }
+        // Clears any checkpoint, this run's included, so next week starts at 0.
+        await saveState(null);
+        return purged;
     }
 
     // -- Public API --------------------------------------------------

src/backend/services/events/EventsService.test.ts

@@ -293,12 +293,14 @@ const buildService = (
     metered: MeteredLine[];
     forwarded: ForwardedCall[];
     eventBus: { on: ReturnType<typeof vi.fn>; emit: ReturnType<typeof vi.fn> };
+    jobs: { register: ReturnType<typeof vi.fn> };
 } => {
     const outbox: Array<{ socket?: string; envelope: DeliveryEnvelope }> = [];
     const counted: DeliveryEnvelope[] = [];
     const lines: MeteredLine[] = [];
     const forwardedCalls: ForwardedCall[] = [];
     const bus = { on: vi.fn(), emit: vi.fn() };
+    const jobs = { register: vi.fn() };
     const built = new EventsService(
         config,
         {
@@ -315,6 +317,7 @@ const buildService = (
             kvShareHandle: kvShareHandleStore,
         } as never,
         {
+            jobs,
             eventForward: {
                 // A deployment with no peers has nowhere to forward to, which
                 // is what every test here is.
@@ -376,6 +379,7 @@ const buildService = (
         metered: lines,
         forwarded: forwardedCalls,
         eventBus: bus,
+        jobs,
     };
 };
 
@@ -564,6 +568,52 @@ describe('the feature switch', () => {
         );
     });
 
+    it('hands its table sweeps to the job scheduler and arms only the pending sweep itself', () => {
+        const intervals = vi.spyOn(globalThis, 'setInterval');
+        const timeouts = vi.spyOn(globalThis, 'setTimeout');
+        let jobs: { register: ReturnType<typeof vi.fn> };
+        try {
+            ({ jobs } = buildService({ events: { enabled: true } } as IConfig));
+            // The pending sweep is the region's queue, so every node runs it.
+            expect(intervals.mock.calls.map(([, ms]) => ms)).toEqual([10_000]);
+            expect(timeouts).not.toHaveBeenCalled();
+        } finally {
+            intervals.mockRestore();
+            timeouts.mockRestore();
+        }
+        expect(
+            jobs.register.mock.calls.map(([name, job]) => ({
+                name,
+                every: job.every,
+                scope: job.scope,
+                nearPrimary: job.nearPrimary ?? false,
+                leaseMs: job.leaseMs,
+            })),
+        ).toEqual([
+            {
+                name: 'events.subscriptions.expirySweep',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.subscriptions.creditSweep',
+                every: 15 * 60_000,
+                scope: 'fleet',
+                nearPrimary: false,
+                leaseMs: 10 * 60_000,
+            },
+        ]);
+    });
+
+    it('registers no job when off', () => {
+        const { jobs } = buildService({
+            events: { enabled: false },
+        } as IConfig);
+        expect(jobs.register).not.toHaveBeenCalled();
+    });
+
     it('spends nothing at all on a dispatch when off', async () => {
         const { service: off } = buildService({
             events: { enabled: false },

src/backend/services/events/appDeletion.integration.test.ts

@@ -29,6 +29,7 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import { EVENTS_WORKER_SESSION_NAME } from './workerRuntime.js';
@@ -327,7 +328,9 @@ describe('an app deleted while it has events state', () => {
         await env.server.stores.app.delete((row as { id: number }).id);
 
         const revoked =
-            await env.server.services.events.sweepStrandedWorkerSessions();
+            await env.server.services.events.sweepStrandedWorkerSessions(
+                testJobContext(),
+            );
         expect(revoked).toBeGreaterThanOrEqual(1);
 
         // `authenticate()` can no longer resolve the token at all — the app
@@ -366,9 +369,60 @@ describe('an app deleted while it has events state', () => {
             EVENTS_WORKER_SESSION_NAME,
         );
 
-        await env.server.services.events.sweepStrandedWorkerSessions();
+        await env.server.services.events.sweepStrandedWorkerSessions(
+            testJobContext(),
+        );
 
         const reauth = await env.server.services.auth.authenticate(token);
         expect(reauth).not.toHaveProperty('reauth');
     });
+
+    it('reaches a stranded session past one run`s two pages on the next run, then wraps', async () => {
+        const db = env.server.clients.db;
+        const now = Math.floor(Date.now() / 1000);
+        const meta = JSON.stringify({
+            worker_name: EVENTS_WORKER_SESSION_NAME,
+        });
+        const workerSession = (appUid: string) => ({
+            statement:
+                'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`, `kind`, `app_uid`) ' +
+                "VALUES (?, ?, ?, ?, ?, 'worker', ?)",
+            values: [uuidv4(), userId, meta, now, now, appUid],
+        });
+        // A run reads two pages of 500; these fill them, all still standing.
+        const live = Array.from(
+            { length: 1000 },
+            (_, i) => `app-live-${uuidv4().slice(0, 8)}-${i}`,
+        );
+        await db.batchWrite(
+            live.map((uid) => ({
+                statement:
+                    'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+                values: [uid, uid, uid, `https://${uid}.example/`, userId],
+            })),
+        );
+        await db.batchWrite(live.map(workerSession));
+        const stranded = workerSession(`app-gone-${uuidv4()}`);
+        await db.batchWrite([stranded]);
+        const strandedRevoked = async () => {
+            const [row] = await db.pread(
+                'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
+                [stranded.values[0]],
+            );
+            return row.revoked_at !== null;
+        };
+
+        const first = testJobContext();
+        await env.server.services.events.sweepStrandedWorkerSessions(first);
+        expect(await strandedRevoked()).toBe(false);
+        expect(first.saved.at(-1)).toMatchObject({
+            afterId: expect.any(Number),
+        });
+
+        const second = testJobContext(first.saved.at(-1));
+        await env.server.services.events.sweepStrandedWorkerSessions(second);
+        expect(await strandedRevoked()).toBe(true);
+        // The end of the table wraps the cursor back to the start.
+        expect(second.saved.at(-1)).toBeNull();
+    });
 });

src/backend/services/events/durable.integration.test.ts

@@ -872,6 +872,71 @@ describe('what a write costs once a durable row exists', () => {
 
         expect(deliveryOf(path)).toBeUndefined();
     });
+
+    it('publishes the expiry reap`s bump, so a region still caching the row drops it', async () => {
+        await clearRows();
+        const created = await subscribe(env.users.user.token, {
+            expiresAt: Math.floor(Date.now() / 1000) + 3600,
+        });
+        const subId = created.body.subId as string;
+        const stale =
+            await env.server.stores.durableSubscription.getBySubId(subId);
+        await env.server.clients.db.write(
+            `UPDATE \`${TABLE}\` SET \`expires_at\` = ? WHERE \`sub_id\` = ?`,
+            [Math.floor(Date.now() / 1000) - 1, subId],
+        );
+
+        const heard: Array<Record<string, unknown>> = [];
+        const listener = (_key: string, data: unknown) => {
+            heard.push(data as Record<string, unknown>);
+        };
+        const announced = vi.spyOn(
+            env.server.services.eventForward,
+            'announceGeneration',
+        );
+        env.server.clients.event.on(
+            'outer.pubsub.events.generationBumped',
+            listener as never,
+        );
+        try {
+            await expect(events().sweepExpired()).resolves.toBe(1);
+            // Over the addressed peer channel as well as the bus.
+            expect(announced).toHaveBeenCalledWith(
+                expect.objectContaining({ userId }),
+                true,
+                'subscription',
+            );
+        } finally {
+            env.server.clients.event.off(
+                'outer.pubsub.events.generationBumped',
+                listener as never,
+            );
+            announced.mockRestore();
+        }
+        const published = heard.find(
+            (bump) => bump.userId === userId && bump.durable === true,
+        );
+        expect(published).toBeDefined();
+
+        // Another region's Redis still holds the row; this one stands in for it.
+        await env.server.stores.eventSubscription.cacheDurable([stale!]);
+        delivered.length = 0;
+        const before = `${anchor}/stale-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: before });
+        await settle(before);
+        expect(deliveryOf(before)?.subId).toBe(subId);
+
+        await env.server.clients.event.emitAndWait(
+            'outer.pubsub.events.generationBumped',
+            published!,
+            { from_outside: true },
+        );
+        delivered.length = 0;
+        const after = `${anchor}/peer-swept-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: after });
+        await quiet();
+        expect(deliveryOf(after)).toBeUndefined();
+    });
 });
 
 describe('where a durable delivery is addressed', () => {

src/backend/services/events/forwarding.test.ts

@@ -79,6 +79,14 @@ vi.mock(import('@opentelemetry/api'), async (importOriginal) => ({
                     metricCalls.push({ name, value, attributes });
                 },
             }),
+            createHistogram: (name: string) => ({
+                record: (
+                    value: number,
+                    attributes: Record<string, unknown> = {},
+                ) => {
+                    metricCalls.push({ name, value, attributes });
+                },
+            }),
         }),
     },
 }));
@@ -441,6 +449,7 @@ const makeRegion = (
             ): Promise<{
                 row: DurableSubscription;
                 bump: { userId: number; generation: number };
+                reaped: [];
             }> => {
                 const row = durableRow({
                     holderUserId: input.holderUserId as number,
@@ -469,6 +478,7 @@ const makeRegion = (
                             row.ownerUserId,
                         ),
                     },
+                    reaped: [],
                 };
             },
         },

src/backend/services/events/handlers.integration.test.ts

@@ -220,7 +220,8 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
-    for (const row of await durable().listActiveForHolder(userId, null))
+    for (const row of (await durable().listActiveForHolderPage(userId, null, 0))
+        .rows)
         await durable().remove(row);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.stores.eventSubscription.rebuildDurable(userId, []);
@@ -777,7 +778,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(404);
         expect(refused.body.code).toBe('events_handler_not_found');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('binds an inline body whose hash is what is published', async () => {
@@ -805,7 +808,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(409);
         expect(refused.body.code).toBe('events_handler_hash_mismatch');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('still needs a name for a subscription owed to one consumer', async () => {

src/backend/services/events/kvHandleRevoke.integration.test.ts

@@ -383,7 +383,7 @@ describe('revoking a handle', () => {
         const sub = await subscribeDurable(handle);
 
         const store = env.server.stores.durableSubscription;
-        const byHolder = vi.spyOn(store, 'listActiveForHolder');
+        const byHolder = vi.spyOn(store, 'listActiveForHolderPage');
         const settle = vi.spyOn(events(), 'settleRevokedGrant');
 
         try {
@@ -400,7 +400,9 @@ describe('revoking a handle', () => {
             for (const call of byHolder.mock.calls)
                 expect(call.slice(1)).toEqual([
                     null,
-                    { includeSuspended: true },
+                    0,
+                    undefined,
+                    { includeSuspended: true, excludeSchedules: true },
                 ]);
         } finally {
             byHolder.mockRestore();
@@ -442,7 +444,9 @@ describe('revoking a handle', () => {
         const page = await shares().listKvHandles(owner.actor, {
             limit: 100,
         });
-        const deeperRow = page.items.find((one) => one.handle === deeper.handle);
+        const deeperRow = page.items.find(
+            (one) => one.handle === deeper.handle,
+        );
         expect(deeperRow?.revokedAt).toBeTypeOf('number');
         expect((await countLive()).total).toBe(before.total);
     });

src/backend/services/events/metering.test.ts

@@ -36,13 +36,17 @@ import {
     DurableQuotaRaceLost,
     type DurableSubscriptionInput,
 } from '../../stores/events/DurableSubscriptionStore.js';
-import type { DurableSubscription } from '../../stores/events/types.js';
+import {
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import {
     DEFAULT_FREE_SUBSCRIPTION,
     DEFAULT_TEMP_SUBSCRIPTION,
 } from '../metering/consts.js';
 import type { UsageInput } from '../metering/types.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_COSTS } from './costs.js';
 import { EventsService } from './EventsService.js';
@@ -62,6 +66,12 @@ let solvent: Set<number>;
 /** Set to have the next create lose the last slot after its insert. */
 let raceLost: boolean;
 let emitted: Array<{ key: string; data: unknown }>;
+/** Rows one credit-sweep page takes; null for the service's own batch. */
+let pageSize: number | null;
+/** Rows one holder page takes; null for all of them in one page. */
+let holderPageSize: number | null;
+/** What `services.schedules` hears about changed schedule rows. */
+let schedulesChanged: (rows: readonly DurableSubscription[]) => void;
 
 const anchorUid = (): string => `docs-${seq}`;
 const anchorPath = (): string => `/u${userId}/Documents`;
@@ -99,7 +109,10 @@ const actorFor = (over: Partial<Actor> = {}): Actor =>
     }) as unknown as Actor;
 
 const appActor = (appUid: string): Actor =>
-    actorFor({ app: { uid: appUid, id: 1 }, effectiveApp: { uid: appUid, id: 1 } });
+    actorFor({
+        app: { uid: appUid, id: 1 },
+        effectiveApp: { uid: appUid, id: 1 },
+    });
 
 const anchorEntry = (): FSEntry =>
     ({
@@ -124,6 +137,9 @@ beforeEach(() => {
     solvent = new Set([userId]);
     raceLost = false;
     emitted = [];
+    pageSize = null;
+    holderPageSize = null;
+    schedulesChanged = () => undefined;
 
     service = new EventsService(
         { events: { enabled: true } } as IConfig,
@@ -155,26 +171,49 @@ beforeEach(() => {
                     return {
                         row,
                         bump: { userId: input.ownerUserId, generation: 1 },
+                        reaped: [],
                     };
                 },
-                listSuspendedPage: async (reason: string, afterId: number) =>
-                    afterId === 0
-                        ? {
-                              rows: rows.filter(
-                                  (row) => row.suspendedReason === reason,
-                              ),
-                              nextId: null,
-                          }
-                        : { rows: [], nextId: null },
-                listSuspendedForHolder: async (
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedPage: async (
+                    reason: string,
+                    afterId: number,
+                    batchSize: number,
+                ) => {
+                    const size = pageSize ?? batchSize;
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId && row.suspendedReason === reason,
+                        )
+                        .slice(0, size);
+                    return {
+                        rows: page.map(({ row }) => row),
+                        nextId: page.length < size ? null : page.at(-1)!.id,
+                    };
+                },
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedForHolderPage: async (
                     holderUserId: number,
                     reason: string,
-                ) =>
-                    rows.filter(
-                        (row) =>
-                            row.holderUserId === holderUserId &&
-                            row.suspendedReason === reason,
-                    ),
+                    afterId: number,
+                ) => {
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId &&
+                                row.holderUserId === holderUserId &&
+                                row.suspendedReason === reason,
+                        );
+                    const size = holderPageSize ?? page.length;
+                    const taken = page.slice(0, size);
+                    return {
+                        rows: taken.map(({ row }) => row),
+                        nextId: page.length > size ? taken.at(-1)!.id : null,
+                    };
+                },
                 resume: async (resuming: readonly DurableSubscription[]) => {
                     for (const row of resuming) {
                         resumed.push(row.subId);
@@ -233,10 +272,7 @@ beforeEach(() => {
             // plan allows, not about who agreed to it.
             permission: { check: async () => true },
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         metered.push({
                             userUuid: actor.user?.uuid,
@@ -247,6 +283,11 @@ beforeEach(() => {
                     solvent.has(Number(actor.user?.id ?? -1)),
                 getActorSubscription: async () => ({ id: plan }),
             },
+            schedules: {
+                onSubscriptionsChanged: async (
+                    changed: readonly DurableSubscription[],
+                ) => schedulesChanged(changed),
+            },
         } as never,
     );
 });
@@ -258,23 +299,105 @@ describe('a suspension waiting on a balance', () => {
             suspendedReason: 'no_credit',
         });
 
-    it('comes back once the account has budget again', async () => {
+    it('comes back once the account has budget again, and every region hears it', async () => {
         const row = suspended();
         rows.push(row);
 
-        await expect(service.sweepNoCredit()).resolves.toBe(1);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(1);
         expect(resumed).toEqual([row.subId]);
         expect(row.suspendedReason).toBeNull();
+        expect(emitted).toContainEqual({
+            key: 'outer.pubsub.events.generationBumped',
+            data: { userId, generation: 1, durable: true },
+        });
     });
 
     it('stays out of service while the account still has none', async () => {
         rows.push(suspended());
         solvent.clear();
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 
+    it('reaches rows past one run`s page cap across runs, then wraps', async () => {
+        // One row per page, so a run's 200 pages stop at 200 rows.
+        pageSize = 1;
+        for (let i = 0; i < 450; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 10_000 + i;
+        // Only the last holder has a balance again.
+        solvent = new Set([10_000 + 449]);
+
+        const first = testJobContext();
+        await expect(service.sweepNoCredit(first)).resolves.toBe(0);
+        expect(first.saved.at(-1)).toEqual({ afterId: 200 });
+
+        const second = testJobContext(first.saved.at(-1));
+        await expect(service.sweepNoCredit(second)).resolves.toBe(0);
+        expect(second.saved.at(-1)).toEqual({ afterId: 400 });
+
+        const third = testJobContext(second.saved.at(-1));
+        await expect(service.sweepNoCredit(third)).resolves.toBe(1);
+        expect(resumed).toEqual([rows[449].subId]);
+        // The end of the table wraps the cursor back to the start.
+        expect(third.saved.at(-1)).toBeNull();
+    });
+
+    it('stops between pages once its signal aborts, keeping the last full page', async () => {
+        pageSize = 1;
+        for (let i = 0; i < 5; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 20_000 + i;
+        const ctx = testJobContext();
+        const save = ctx.saveState;
+        ctx.saveState = async (next) => {
+            await save(next);
+            if (ctx.saved.length === 2) ctx.abort();
+        };
+
+        await service.sweepNoCredit(ctx);
+        expect(ctx.saved).toEqual([{ afterId: 1 }, { afterId: 2 }]);
+    });
+
+    it('resumes no later holder page once its signal aborts inside a page`s hook, keeping the checkpoint', async () => {
+        // A full sweep page, so finishing it would move the checkpoint.
+        pageSize = 3;
+        holderPageSize = 1;
+        for (let i = 0; i < 3; i++)
+            rows.push(
+                durableRow({
+                    subId: `sched-${i}`,
+                    token: scheduleToken(`schedule-${seq}-${i}`),
+                    suspendedAt: Math.floor(Date.now() / 1000),
+                    suspendedReason: 'no_credit',
+                }),
+            );
+        const ctx = testJobContext();
+        const heard: string[][] = [];
+        schedulesChanged = (changed) => {
+            heard.push(changed.map((row) => row.subId));
+            ctx.abort();
+        };
+
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(1);
+        expect(resumed).toEqual(['sched-0']);
+        expect(heard).toEqual([['sched-0']]);
+        expect(rows[1].suspendedReason).toBe('no_credit');
+        expect(ctx.saved).toEqual([]);
+    });
+
+    it('wraps a saved cursor past the end of a shrunken table back to the start', async () => {
+        rows.push(suspended());
+        const ctx = testJobContext({ afterId: 999 });
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([null]);
+    });
+
+    it('writes no cursor while there is nothing to page through', async () => {
+        const ctx = testJobContext();
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([]);
+    });
+
     it('leaves a suspension it did not cause alone', async () => {
         rows.push(
             durableRow({
@@ -283,7 +406,7 @@ describe('a suspension waiting on a balance', () => {
             }),
         );
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 });
@@ -298,9 +421,10 @@ describe('what a plan lets an account subscribe to', () => {
         await subscribe(appActor(`app-${seq}`));
 
         expect(created[0].limits).toEqual({
-            perUser: EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
-                DEFAULT_FREE_SUBSCRIPTION
-            ],
+            perUser:
+                EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ],
             perApp: EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],

src/backend/services/events/registry.test.ts

@@ -28,6 +28,7 @@ import {
     lookupFsSubject,
     lookupKvSubject,
     lookupPublicSubject,
+    projectScheduleTick,
     type FsDeliveryContext,
     type FsPublicSubject,
     type KvDeliveryContext,
@@ -324,3 +325,52 @@ describe('registry typing', () => {
         expect(subjects.length).toBeGreaterThan(0);
     });
 });
+
+describe('the schedule tick', () => {
+    const tick = {
+        scheduleId: '2f7c1b6e-4d1a-4c55-9a8e-0e6f3c2b1a90',
+        scheduledFor: 1_798_761_600_000,
+        missed: 0,
+        missedCapped: false,
+        attempt: 1,
+    };
+
+    it('carries exactly the fields a handler is promised', () => {
+        const event = projectScheduleTick(tick);
+        expect(Object.keys(event).sort()).toEqual([
+            'attempt',
+            'id',
+            'missed',
+            'op',
+            'scheduleId',
+            'scheduledFor',
+            'subject',
+        ]);
+        expect(event).toEqual({
+            subject: `schedule:${tick.scheduleId}`,
+            op: 'tick',
+            id: `${tick.scheduleId}:${tick.scheduledFor}`,
+            scheduleId: tick.scheduleId,
+            scheduledFor: tick.scheduledFor,
+            missed: 0,
+            attempt: 1,
+        });
+    });
+
+    it('says when `missed` stopped counting, and only then', () => {
+        const capped = projectScheduleTick({
+            ...tick,
+            missed: 1000,
+            missedCapped: true,
+        });
+        expect(capped.missedCapped).toBe(true);
+        expect(capped.missed).toBe(1000);
+        expect('missedCapped' in projectScheduleTick(tick)).toBe(false);
+    });
+
+    it('keeps the id across attempts at one slot', () => {
+        expect(projectScheduleTick({ ...tick, attempt: 3 }).id).toBe(
+            projectScheduleTick(tick).id,
+        );
+    });
+});

src/backend/services/events/registry.ts

@@ -20,6 +20,7 @@
 import type { EventKey, KvOp } from '../../clients/event/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import type { AclMode } from '../acl/ACLService.js';
+import { scheduleToken } from '../../stores/events/types.js';
 import type { NotificationAudience } from '../notification/notificationTypes.js';
 import { relativeTo } from './matcher.js';
 import {
@@ -359,6 +360,50 @@ const notifProject = (delivery: NotifDeliveryContext): ProjectedNotifEvent => ({
     seq: delivery.seq,
 });
 
+// -- Schedule ticks ---------------------------------------------------
+
+/**
+ * What a schedule's handler is handed as `event`. No bus event stands behind
+ * it, so it has no registry entry: the runner builds it from the slot it fires.
+ * Additive only, as for the other projections.
+ */
+export interface ScheduleEvent {
+    /** `schedule:<scheduleId>`. */
+    subject: string;
+    op: 'tick';
+    /** `${scheduleId}:${scheduledFor}`: the same across retries of one slot. */
+    id: string;
+    scheduleId: string;
+    /** The slot this run is for, in epoch ms. */
+    scheduledFor: number;
+    /** When the invocation was sent; stamped as it goes out. */
+    firedAt?: number;
+    /** Slots skipped since the last one that ran. */
+    missed: number;
+    /** Set when `missed` stopped counting at its cap. */
+    missedCapped?: true;
+    /** Attempts at this slot, this one included. */
+    attempt: number;
+}
+
+/** The {@link ScheduleEvent} of one run. Called by extensions. */
+export const projectScheduleTick = (tick: {
+    scheduleId: string;
+    scheduledFor: number;
+    missed: number;
+    missedCapped: boolean;
+    attempt: number;
+}): ScheduleEvent => ({
+    subject: scheduleToken(tick.scheduleId),
+    op: 'tick',
+    id: `${tick.scheduleId}:${tick.scheduledFor}`,
+    scheduleId: tick.scheduleId,
+    scheduledFor: tick.scheduledFor,
+    missed: tick.missed,
+    ...(tick.missedCapped ? { missedCapped: true as const } : {}),
+    attempt: tick.attempt,
+});
+
 // -- Registry ---------------------------------------------------------
 
 export const PUBLIC_SUBJECTS = [

src/backend/services/events/requestParsing.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { HttpError } from '../../core/http/HttpError.js';
+import { checkRateLimit } from '../../core/http/middleware/rateLimit.js';
+import type { RouteRateLimit } from '../../core/http/types.js';
+import {
+    HANDLER_NAME_MAX_LENGTH,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+
+/** Request and response pieces the subscription and schedule verbs share. */
+
+export const badRequest = (message: string, code: string): HttpError =>
+    new HttpError(400, message, { legacyCode: code });
+
+export const parseHandlerName = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || value.length === 0)
+        throw badRequest('handlerName must be a string', 'bad_request');
+    if (value.length > HANDLER_NAME_MAX_LENGTH)
+        throw badRequest(
+            `handlerName may not exceed ${HANDLER_NAME_MAX_LENGTH} characters`,
+            'bad_request',
+        );
+    return value;
+};
+
+/** Hex digest of the inline source a subscribe claims it is binding. */
+export const parseHandlerHash = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value))
+        throw badRequest(
+            'handlerHash must be a sha-256 hex digest',
+            'bad_request',
+        );
+    return value;
+};
+
+/** Stored as JSON text; the byte cap is the store's to enforce. */
+export const parseContext = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    try {
+        return JSON.stringify(value);
+    } catch {
+        throw badRequest('context must be JSON-serializable', 'bad_request');
+    }
+};
+
+const tooManyCalls = (): HttpError =>
+    new HttpError(429, 'Too many subscription changes', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** Spend one call from a per-user window, refusing past it. */
+export const spendUserWindow = async (
+    limit: RouteRateLimit,
+    userId: number,
+): Promise<void> => {
+    const ok = await checkRateLimit(
+        `${limit.scope}:${userId}`,
+        limit.limit,
+        limit.window,
+    );
+    if (!ok) throw tooManyCalls();
+};
+
+/**
+ * A stored context as a listing may describe it: which keys it sets, and a hash
+ * of the whole blob. Never the values: the column is where an API key lives.
+ */
+export const projectContext = (
+    context: string | null,
+): { contextKeys: string[] | null; contextHash: string | null } => {
+    if (context === null) return { contextKeys: null, contextHash: null };
+    let keys: string[] = [];
+    try {
+        const parsed: unknown = JSON.parse(context);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+            keys = Object.keys(parsed).sort();
+    } catch {
+        // Stored as JSON by the verbs, so this cannot normally happen; an
+        // unreadable blob still reports its hash rather than failing the list.
+    }
+    return { contextKeys: keys, contextHash: hashContent(context) };
+};

src/backend/services/events/scheduleSeam.integration.test.ts

@@ -0,0 +1,344 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The subscription lifecycle hands schedules' rows, and only theirs, to
+ * `services.schedules` with the change each path made. Without it every path
+ * still runs.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type {
+    ScheduleServiceContract,
+    ScheduleSubscriptionChange,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import type { DurableSubscriptionInput } from '../../stores/events/DurableSubscriptionStore.js';
+import {
+    isScheduleRow,
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
+
+const BOOT_TIMEOUT_MS = 120_000;
+const APP_UID = 'app-seam';
+
+let server: PuterServer;
+let userId: number;
+let anchorUid: string;
+let anchorPath: string;
+let saved: unknown;
+
+const durable = () => server.stores.durableSubscription;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const recording = () => {
+    const calls: Array<{
+        subIds: string[];
+        change: ScheduleSubscriptionChange;
+    }> = [];
+    const schedules = {
+        onSubscriptionsChanged: vi.fn(
+            async (
+                rows: readonly DurableSubscription[],
+                change: ScheduleSubscriptionChange,
+            ) => {
+                calls.push({ subIds: rows.map((r) => r.subId), change });
+            },
+        ),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    } satisfies ScheduleServiceContract;
+    setSchedules(schedules);
+    return calls;
+};
+
+const plainRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> =>
+    (
+        await durable().create({
+            holderUserId: userId,
+            ownerUserId: userId,
+            appUid: APP_UID,
+            subject: `fs:${anchorPath}`,
+            token: `f#${anchorUid}`,
+            anchorUid,
+            anchorPath,
+            match: null,
+            op: null,
+            delivery: 'broadcast',
+            handlerName: 'onWrite',
+            context: null,
+            permission: 'list',
+            expiresAt: null,
+            ...over,
+        })
+    ).row;
+
+const scheduleRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> => {
+    const uid = uuidv4();
+    return plainRow({
+        subject: scheduleToken(uid),
+        token: scheduleToken(uid),
+        anchorUid: uid,
+        anchorPath: '',
+        delivery: 'single',
+        handlerName: 'tick',
+        permission: 'events:background',
+        ...over,
+    });
+};
+
+const pendingCalls = () => {
+    const pending = server.stores.pendingDelivery;
+    return [
+        vi.spyOn(pending, 'purge'),
+        vi.spyOn(pending, 'hold'),
+        vi.spyOn(pending, 'releaseHold'),
+    ];
+};
+
+beforeAll(async () => {
+    server = await setupTestServer({ events: { enabled: true } } as IConfig);
+    const username = `seam-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+    });
+    userId = user.id;
+    anchorPath = `/${username}/seam`;
+    await server.services.fs.mkdir(userId, {
+        path: anchorPath,
+        createMissingParents: true,
+    });
+    anchorUid = (await server.stores.fsEntry.getEntryByPath(anchorPath))!.uid;
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await server.clients.db.write('DELETE FROM `event_subscriptions`', []);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('the schedules lifecycle hook', () => {
+    it('gets only schedule rows on a suspend, and their backlog is never touched', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const [purge, hold, releaseHold] = pendingCalls();
+
+        await server.services.events.suspendSubscriptions(
+            [schedule],
+            'no_credit',
+        );
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'no_credit' },
+            },
+        ]);
+        expect(purge).not.toHaveBeenCalled();
+        expect(hold).not.toHaveBeenCalled();
+
+        calls.length = 0;
+        await server.services.events.suspendSubscriptions([plain], 'no_credit');
+        expect(calls).toEqual([]);
+
+        calls.length = 0;
+        const suspended = (await durable().getBySubId(schedule.subId))!;
+        await server.services.events.resumeSubscriptions([suspended]);
+        expect(calls).toEqual([
+            { subIds: [schedule.subId], change: { type: 'resumed' } },
+        ]);
+        expect(releaseHold).not.toHaveBeenCalled();
+    });
+
+    it('ends schedules uninstalled when every grant to the app goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: null,
+        });
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: {
+                    type: 'suspended',
+                    reason: 'permission_revoked',
+                    endReason: 'uninstalled',
+                },
+            },
+        ]);
+    });
+
+    it('reads no schedule rows to settle a specific grant, and still settles them when consent goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const page = vi.spyOn(durable(), 'listActiveForHolderPage');
+        const readSubIds = async () =>
+            (await Promise.all(page.mock.results.map((r) => r.value))).flatMap(
+                (p) => p.rows.map((row: DurableSubscription) => row.subId),
+            );
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: `fs:${anchorUid}:list`,
+        });
+        const read = await readSubIds();
+        expect(read).toContain(plain.subId);
+        expect(read).not.toContain(schedule.subId);
+        expect(calls).toEqual([]);
+
+        page.mockClear();
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: EVENTS_BACKGROUND_PERMISSION,
+        });
+        expect(await readSubIds()).toContain(schedule.subId);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'permission_revoked' },
+            },
+        ]);
+    });
+
+    it('tells the holder once per kind when an uninstall settles several pages', async () => {
+        recording();
+        const plains = [await plainRow(), await plainRow(), await plainRow()];
+        const schedules = [await scheduleRow(), await scheduleRow()];
+        const store = durable();
+        const listPage = store.listActiveForHolderPage.bind(store);
+        vi.spyOn(store, 'listActiveForHolderPage').mockImplementation(
+            (holder, app, afterId, _batchSize, opts) =>
+                listPage(holder, app, afterId, 1, opts),
+        );
+        const notify = vi.spyOn(server.services.notification, 'notify');
+
+        await expect(
+            server.services.events.settleRevokedGrant({
+                holderUserId: userId,
+                appUid: APP_UID,
+                permission: null,
+            }),
+        ).resolves.toBe(5);
+        const ended = notify.mock.calls
+            .filter(([, , meta]) => meta?.type === 'app.events.ended')
+            .map(([, body]) => body as { count: number; subjects: string[] });
+        expect(ended).toHaveLength(2);
+        expect(ended.map((body) => body.count).sort()).toEqual([2, 3]);
+        expect(ended.flatMap((body) => body.subjects).sort()).toEqual(
+            [...plains, ...schedules].map((row) => row.subject).sort(),
+        );
+    });
+
+    it('ends schedules app_deleted when their app is deleted', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleDeletedApp(APP_UID);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'ended', endReason: 'app_deleted' },
+            },
+        ]);
+    });
+
+    it('ends schedules whose rows expire', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow({ expiresAt: 1 });
+        await plainRow({ expiresAt: 1 });
+
+        await server.services.events.sweepExpired();
+        const handed = calls.flatMap((c) => c.subIds);
+        expect(handed).toEqual([schedule.subId]);
+        expect(calls[0].change).toEqual({ type: 'ended', endReason: 'ended' });
+    });
+
+    it('lets every path run without services.schedules', async () => {
+        setSchedules(undefined);
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await expect(
+            server.services.events.suspendSubscriptions(
+                [schedule],
+                'no_credit',
+            ),
+        ).resolves.toBe(1);
+        await expect(
+            server.services.events.resumeSubscriptions([
+                (await durable().getBySubId(schedule.subId))!,
+            ]),
+        ).resolves.toBe(1);
+        await server.services.events.settleDeletedApp(APP_UID);
+        const left = await server.clients.db.read(
+            'SELECT `token` FROM `event_subscriptions`',
+            [],
+        );
+        expect(
+            left.filter((r) => isScheduleRow(r as { token: string })),
+        ).toEqual([]);
+    });
+});

src/backend/services/events/workerSeam.ts

@@ -17,8 +17,14 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import type { EventsWorkerInvokerClient } from '../../clients/events/EventsWorkerInvokerClient.js';
-import type { DeliverableEvent } from './registry.js';
+import type {
+    EventsWorkerInvokerClient,
+    WorkerInvokeOptions,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
+import type { Actor } from '../../core/actor.js';
+import type { DeliverableEvent, ScheduleEvent } from './registry.js';
+
+export type { ScheduleEvent } from './registry.js';
 
 /**
  * Where a delivery leaves the event system for the app's own code.
@@ -36,11 +42,13 @@ export interface WorkerInvocation {
     /** The app whose handler runs, or null for an account-owned row. */
     appUid: string | null;
     handlerName: string | null;
-    event: DeliverableEvent;
+    event: DeliverableEvent | ScheduleEvent;
     /** The subscription's stored context, delivered to the handler as `ctx`. */
     context: string | null;
     /** Handler runs behind `event`; the token this run is handed is one deeper. */
     handlerDepth: number;
+    /** The holder acting as the app, when the caller resolved it already. */
+    actor?: Actor;
 }
 
 /**
@@ -50,7 +58,11 @@ export interface WorkerInvocation {
  * toward the run that suspends a subscription.
  */
 export type WorkerInvocationOutcome =
-    'settled' | 'terminal' | 'retriable' | 'unavailable' | 'deferred';
+    | 'settled'
+    | 'terminal'
+    | 'retriable'
+    | 'unavailable'
+    | 'deferred';
 
 export interface WorkerInvocationResult {
     outcome: WorkerInvocationOutcome;
@@ -64,6 +76,12 @@ export interface WorkerInvokerSeam {
     available?(): boolean;
 }
 
+/**
+ * Why no token could be minted: nobody to run as, no consent to, or the mint
+ * itself failed.
+ */
+export type MintRefusal = 'no-identity' | 'no-consent' | 'mint-failed';
+
 /**
  * Mints the subscriber's app-under-user token for one invocation. The
  * `events:background` consent is what authorizes it, not a narrower
@@ -71,7 +89,14 @@ export interface WorkerInvokerSeam {
  */
 export type SubscriberTokenMinter = (
     invocation: WorkerInvocation,
-) => Promise<string | null>;
+) => Promise<{ token: string } | { refused: MintRefusal }>;
+
+/** Where the handler set is read from when its cache misses. */
+export interface AddressOptions {
+    missRead?: 'primary' | 'replica';
+    /** A handler the set must have; a set without it is re-read on the primary. */
+    handlerName?: string;
+}
 
 /**
  * The script an app's handlers deploy as and its invocation key; null when the
@@ -79,8 +104,17 @@ export type SubscriberTokenMinter = (
  */
 export type EventsWorkerAddresser = (
     appUid: string,
+    options?: AddressOptions,
 ) => Promise<{ script: string; key: string } | null>;
 
+/** What one run came to, including why nothing was invoked. */
+export type WorkerRunResult =
+    | WorkerInvocationResult
+    | { outcome: MintRefusal | 'deploy-busy' };
+
+/** How {@link EventsWorkerInvoker.run} reaches the worker. */
+export type WorkerRunOptions = WorkerInvokeOptions & AddressOptions;
+
 /** How often the same script's retriable failures are logged. */
 const WARN_INTERVAL_MS = 60_000;
 /** Distinct scripts one process tracks a last-warned time for. */
@@ -122,28 +156,57 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
     async invoke(
         invocation: WorkerInvocation,
     ): Promise<WorkerInvocationResult> {
+        const result = await this.run(invocation);
+        switch (result.outcome) {
+            // Nothing was attempted.
+            case 'no-identity':
+            case 'no-consent':
+            case 'mint-failed':
+                return { outcome: 'deferred' };
+            case 'deploy-busy':
+                return { outcome: 'unavailable' };
+            default:
+                return result;
+        }
+    }
+
+    /** {@link invoke}, saying why nothing was invoked. */
+    async run(
+        invocation: WorkerInvocation,
+        options: WorkerRunOptions = {},
+    ): Promise<WorkerRunResult> {
         const { appUid, handlerName } = invocation;
         if (!appUid || !handlerName) return { outcome: 'deferred' };
         if (!this.#available()) return { outcome: 'unavailable' };
 
-        const token = await this.#mintToken(invocation);
-        if (token === null) return { outcome: 'deferred' };
+        const minted = await this.#mintToken(invocation);
+        if ('refused' in minted) return { outcome: minted.refused };
 
         // Nowhere to send it: the app has no handlers or no owner to run them
         // as. Retrying cannot fix that, so the consecutive-failure rule is what
         // stops it.
-        const address = await this.#address(appUid);
+        const address = await this.#address(appUid, {
+            missRead: options.missRead,
+            handlerName,
+        });
         if (address === null) return { outcome: 'retriable' };
 
-        const result = await this.#client.invoke({
-            script: address.script,
-            key: address.key,
-            appUid,
-            handler: handlerName,
-            token,
-            event: invocation.event,
-            ctx: parseContext(invocation.context),
-        });
+        const result = await this.#client.invoke(
+            {
+                script: address.script,
+                key: address.key,
+                appUid,
+                handler: handlerName,
+                token: minted.token,
+                event: invocation.event,
+                ctx: parseContext(invocation.context),
+            },
+            {
+                lane: options.lane,
+                stampEvent: options.stampEvent,
+            },
+        );
+        if (result.deployBusy) return { outcome: 'deploy-busy' };
         if (
             (result.outcome === 'retriable' ||
                 result.outcome === 'unavailable') &&

src/backend/services/index.ts

@@ -36,6 +36,7 @@ import { FSService } from './fs/FSService';
 import { KvShareService } from './kvShare/KvShareService';
 import { ServerHealthService } from './health/ServerHealthService';
 import { PuterHomepageService } from './homepage/PuterHomepageService';
+import { JobScheduler } from './jobs/JobScheduler';
 import { LocalWorkerService } from './localworker/LocalWorkerService';
 import { MeteringService } from './metering/MeteringService';
 import { NotificationService } from './notification/NotificationService';
@@ -60,6 +61,7 @@ import { UserAccountService } from './user/UserAccountService';
  */
 declare module './types' {
     interface IPuterServiceInstances {
+        jobs: JobScheduler;
         metering: MeteringService;
         appOriginBlocklist: AppOriginBlocklistService;
         permission: PermissionService;
@@ -103,6 +105,9 @@ declare module './types' {
 // NotificationService depends on notification store (for DB) + event client (for socket push).
 // BroadcastService is independent — only needs the event client.
 export const puterServices = {
+    // First, so every later service can register jobs with it. Reaches
+    // `broadcast` at call time only.
+    jobs: JobScheduler,
     metering: MeteringService,
     // Declared before `auth` so AuthService sees it as a prior peer — it
     // queries the blocklist on app-token acquisition and per-request app

src/backend/services/jobs/JobScheduler.test.ts

@@ -0,0 +1,781 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import MockRedis from 'ioredis-mock';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobLeaseLostError,
+    JobScheduler,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const HOUR = 60 * 60_000;
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+describe('JobScheduler', () => {
+    let server: PuterServer;
+    const schedulers: JobScheduler[] = [];
+
+    beforeAll(async () => {
+        server = await setupTestServer();
+    });
+
+    afterAll(async () => {
+        await server?.shutdown();
+    });
+
+    afterEach(() => {
+        vi.restoreAllMocks();
+        vi.useRealTimers();
+        for (const scheduler of schedulers.splice(0)) {
+            scheduler.onServerPrepareShutdown();
+        }
+    });
+
+    const db = () => server.clients.db;
+
+    /**
+     * A node of its own: its own config and region, on the test server's Redis
+     * unless given one.
+     */
+    const node = ({
+        namespace,
+        region = 'r1',
+        redis,
+        domain = 'puter.test',
+        enabled,
+    }: {
+        namespace?: string;
+        region?: string;
+        redis?: InstanceType<typeof MockRedis>;
+        domain?: string;
+        enabled?: boolean;
+    } = {}) => {
+        const config = {
+            port: 0,
+            extensions: [],
+            domain,
+            serverId: `node-${schedulers.length}`,
+            jobs: { namespace, enabled },
+        } as IConfig;
+        const clients = (
+            redis ? { ...server.clients, redis } : server.clients
+        ) as SchedulerArgs[1];
+        const services = {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3];
+        const scheduler = new JobScheduler(
+            config,
+            clients,
+            server.stores as SchedulerArgs[2],
+            services,
+        );
+        schedulers.push(scheduler);
+        return scheduler;
+    };
+
+    const freshNamespace = () => `ns-${randomUUID()}`;
+
+    /**
+     * A job that runs `batches` batches `batchMs` apart, checking its signal
+     * before each.
+     */
+    const batchedJob = (batches: number, batchMs: number) => {
+        const seen = { done: 0, signal: null as AbortSignal | null };
+        const run = async ({ signal }: JobRunContext) => {
+            seen.signal = signal;
+            for (let i = 0; i < batches; i++) {
+                if (signal.aborted) return;
+                await sleep(batchMs);
+                seen.done += 1;
+            }
+        };
+        return { seen, run };
+    };
+
+    const keysOf = (namespace: string, name: string) => ({
+        lock: `jobs:{${namespace}:${name}}:lock`,
+        gate: `jobs:{${namespace}:${name}}:next`,
+        run: `jobs:{${namespace}:${name}}:run`,
+    });
+
+    // -- Registration -------------------------------------------------
+
+    it('refuses a bad or duplicate registration', () => {
+        const scheduler = node({ namespace: freshNamespace() });
+        const run = async () => {};
+        expect(() =>
+            scheduler.register('bad name', {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job', { every: 0, scope: 'fleet', run }),
+        ).toThrow(/every/);
+        expect(() =>
+            scheduler.register('job', {
+                every: 1000,
+                scope: 'fleet',
+                leaseMs: 1.5,
+                run,
+            }),
+        ).toThrow(/leaseMs/);
+        // `:` would let two (namespace, name) pairs share Redis keys.
+        expect(() =>
+            scheduler.register('b:c', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('x'.repeat(129), {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job.region', {
+                every: 1000,
+                scope: 'region',
+                leaseMs: 2000,
+                run,
+            }),
+        ).toThrow(/exceeds every/);
+        scheduler.register('job', { every: 1000, scope: 'fleet', run });
+        expect(() =>
+            scheduler.register('job', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/already registered/);
+    });
+
+    it('falls back to the default namespace, once and loudly, when the configured one cannot be held', () => {
+        const run = async () => {};
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        for (const namespace of ['a:b', 'x'.repeat(129), 'spaced name']) {
+            const scheduler = node({ namespace });
+            scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+            expect(scheduler.namespace).toBe('default');
+        }
+        logged.mockClear();
+
+        // A self-host whose domain carries a port still boots.
+        const scheduler = node({ domain: 'localhost:4100' });
+        scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+        scheduler.register('job.b', { every: 1000, scope: 'fleet', run });
+        expect(scheduler.namespace).toBe('default');
+        expect(logged).toHaveBeenCalledTimes(1);
+        expect(String(logged.mock.calls[0][0])).toMatch(
+            /localhost:4100.*jobs\.namespace/,
+        );
+    });
+
+    it('does nothing at all while no job is registered', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        // Timers the test runtime itself already holds.
+        const baseline = vi.getTimerCount();
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        const scheduler = node({ namespace: freshNamespace() });
+
+        scheduler.onServerStart();
+        expect(vi.getTimerCount()).toBe(baseline);
+        await scheduler.poll();
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+
+        // A job registered after start arms the poll; shutdown disarms it.
+        scheduler.register('late', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {},
+        });
+        expect(vi.getTimerCount()).toBeGreaterThan(baseline);
+        scheduler.onServerPrepareShutdown();
+        expect(vi.getTimerCount()).toBe(baseline);
+    });
+
+    it('claims nothing once draining, and leaves a running job its lease', async () => {
+        const scheduler = node({ namespace: `drain${Date.now()}` });
+        let release!: () => void;
+        let signal!: AbortSignal;
+        scheduler.register('long', {
+            every: HOUR,
+            scope: 'region',
+            run: (ctx) => {
+                signal = ctx.signal;
+                return new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+            },
+        });
+        const first = scheduler.poll();
+        await vi.waitFor(() => expect(release).toBeDefined());
+        const run = vi.fn(async () => {});
+        scheduler.register('other', { every: HOUR, scope: 'region', run });
+
+        scheduler.onServerDrainStart();
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(signal.aborted).toBe(false);
+
+        release();
+        await first;
+    });
+
+    it('arms no poll on a node with jobs turned off, though poll() still runs them', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        const baseline = vi.getTimerCount();
+        const scheduler = node({ namespace: freshNamespace(), enabled: false });
+        const warned = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let runs = 0;
+        scheduler.onServerStart();
+        scheduler.register('job.off', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {
+                runs += 1;
+            },
+        });
+        scheduler.register('job.off.too', {
+            every: HOUR,
+            scope: 'region',
+            run: async () => undefined,
+        });
+        expect(vi.getTimerCount()).toBe(baseline);
+        // Said once per node, not per job.
+        expect(
+            warned.mock.calls.filter(([line]) =>
+                String(line).includes('jobs.enabled'),
+            ),
+        ).toHaveLength(1);
+        vi.useRealTimers();
+
+        await scheduler.poll();
+        expect(runs).toBe(1);
+    });
+
+    describe('on a controlled clock', () => {
+        const leaseMs = 300;
+        /**
+         * Pending timeouts armed from JobScheduler.ts. The shared test server's
+         * own loops keep running on real intervals and may arm fake timers
+         * mid-test, so the global timer count says nothing about the
+         * scheduler.
+         */
+        let armed: Set<unknown>;
+        const schedulerTimers = () => armed.size;
+
+        beforeEach(() => {
+            vi.useFakeTimers({
+                toFake: [
+                    'setTimeout',
+                    'clearTimeout',
+                    'setInterval',
+                    'clearInterval',
+                    'performance',
+                ],
+            });
+            armed = new Set();
+            const fakeSetTimeout = globalThis.setTimeout;
+            const fakeClearTimeout = globalThis.clearTimeout;
+            const fromScheduler = () => {
+                const limit = Error.stackTraceLimit;
+                Error.stackTraceLimit = 30;
+                const stack = new Error().stack ?? '';
+                Error.stackTraceLimit = limit;
+                return /services[\\/]jobs[\\/]JobScheduler\.ts/.test(stack);
+            };
+            vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+                fn: (...args: unknown[]) => void,
+                ms?: number,
+                ...rest: unknown[]
+            ) => {
+                if (!fromScheduler()) return fakeSetTimeout(fn, ms, ...rest);
+                const handle = fakeSetTimeout(
+                    (...args: unknown[]) => {
+                        armed.delete(handle);
+                        fn(...args);
+                    },
+                    ms,
+                    ...rest,
+                );
+                armed.add(handle);
+                return handle;
+            }) as never);
+            vi.spyOn(globalThis, 'clearTimeout').mockImplementation(((
+                handle: Parameters<typeof clearTimeout>[0],
+            ) => {
+                armed.delete(handle);
+                fakeClearTimeout(handle);
+            }) as never);
+        });
+
+        /** Batches of `batchMs`, each started only while the signal is live. */
+        const timedJob = (batches: number, batchMs: number) => {
+            const seen = {
+                starts: [] as number[],
+                abortedAt: null as number | null,
+                signal: null as AbortSignal | null,
+                saveError: null as unknown,
+            };
+            const run = async ({ signal, saveState }: JobRunContext) => {
+                seen.signal = signal;
+                signal.addEventListener('abort', () => {
+                    seen.abortedAt = performance.now();
+                });
+                for (let i = 0; i < batches; i++) {
+                    if (signal.aborted) break;
+                    seen.starts.push(performance.now());
+                    await sleep(batchMs);
+                }
+                try {
+                    await saveState({ late: true });
+                } catch (err) {
+                    seen.saveError = err;
+                }
+            };
+            return { seen, run };
+        };
+
+        /** Let the run play out, then check nothing it armed is left behind. */
+        const settle = async (running: Promise<void>) => {
+            await vi.advanceTimersByTimeAsync(2_000);
+            await running;
+            expect(schedulerTimers()).toBe(0);
+        };
+
+        it('clears a run’s timers at shutdown even while the run hangs', async () => {
+            const scheduler = node({ namespace: freshNamespace() });
+            let release!: () => void;
+            scheduler.register('job.hang', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: () =>
+                    new Promise<void>((resolve) => {
+                        release = resolve;
+                    }),
+            });
+            const running = scheduler.poll();
+            await vi.advanceTimersByTimeAsync(10);
+            expect(schedulerTimers()).toBeGreaterThan(0);
+
+            scheduler.onServerPrepareShutdown();
+            expect(schedulerTimers()).toBe(0);
+            release();
+            await settle(running);
+        });
+
+        it('never starts a region run once taking the lock used up half the lease', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17104 });
+            const scheduler = node({ namespace, redis });
+            const run = vi.fn(async () => {});
+            scheduler.register('job.slowLock', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run,
+            });
+            const set = redis.set.bind(redis) as (
+                ...args: unknown[]
+            ) => Promise<unknown>;
+            const slowLock = vi.spyOn(redis, 'set').mockImplementation((async (
+                ...args: unknown[]
+            ) => {
+                const result = await set(...args);
+                if (String(args[0]).endsWith(':lock')) {
+                    await sleep(leaseMs / 2 + 10);
+                }
+                return result;
+            }) as never);
+            await settle(scheduler.poll());
+
+            expect(run).not.toHaveBeenCalled();
+            // The gate went back, so the next poll runs it.
+            expect(
+                await redis.exists(`jobs:{${namespace}:job.slowLock}:next`),
+            ).toBe(0);
+            slowLock.mockRestore();
+            await settle(scheduler.poll());
+            expect(run).toHaveBeenCalledTimes(1);
+        });
+
+        it('aborts a region run before its next batch once the lock lapses, and leaves it due at once', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17106 });
+            const scheduler = node({ namespace, redis });
+            const job = timedJob(10, 70);
+            scheduler.register('job.lapse', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: job.run,
+            });
+            const lockedAt = performance.now();
+            await settle(scheduler.poll());
+
+            expect(job.seen.abortedAt! - lockedAt).toBe(leaseMs);
+            expect((job.seen.signal?.reason as Error).message).toMatch(
+                /lapsed/,
+            );
+            expect(job.seen.starts.length).toBeLessThan(10);
+            expect(job.seen.starts.every((t) => t < job.seen.abortedAt!)).toBe(
+                true,
+            );
+            // The gate is still this run's, so its last save went through;
+            // interrupted, the gate goes back and the job is due at once.
+            expect(job.seen.saveError).toBeNull();
+            const keys = {
+                gate: `jobs:{${namespace}:job.lapse}:next`,
+                run: `jobs:{${namespace}:job.lapse}:run`,
+            };
+            expect(await redis.exists(keys.gate)).toBe(0);
+            expect(await redis.hget(keys.run, 'state')).toBe('{"late":true}');
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        });
+    });
+
+    it('refuses state over 64 KB', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17107 });
+        const scheduler = node({ namespace, redis });
+        let refused: unknown = null;
+        scheduler.register('job.bigState', {
+            every: HOUR,
+            scope: 'region',
+            run: async ({ saveState }) => {
+                try {
+                    await saveState({ blob: 'x'.repeat(70_000) });
+                } catch (err) {
+                    refused = err;
+                }
+            },
+        });
+        await scheduler.poll();
+        expect((refused as Error).message).toMatch(/over 65536 bytes/);
+        expect(
+            await redis.hget(keysOf(namespace, 'job.bigState').run, 'state'),
+        ).toBeNull();
+    });
+
+    it('releases an interrupted run on shutdown without counting a failure', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17109 });
+        const scheduler = node({ namespace, redis });
+        const job = batchedJob(50, 50);
+        scheduler.register('job.shutdown', {
+            every: HOUR,
+            scope: 'region',
+            run: job.run,
+        });
+        const running = scheduler.poll();
+        await vi.waitFor(() => expect(job.seen.done).toBeGreaterThan(0));
+        scheduler.onServerPrepareShutdown();
+        await running;
+
+        expect(job.seen.done).toBeLessThan(50);
+        expect((job.seen.signal?.reason as Error).message).toBe(
+            'server shutting down',
+        );
+        const keys = keysOf(namespace, 'job.shutdown');
+        // Due again at once, with no failure counted.
+        expect(await redis.exists(keys.gate)).toBe(0);
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+
+    it('keeps namespaces apart: the same job name runs once in each', async () => {
+        const redis = new MockRedis({ port: 17110 });
+        const [nsA, nsB] = [freshNamespace(), freshNamespace()];
+        const nodes = [
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsB, redis }),
+            node({ namespace: nsB, redis }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.shared', {
+                every: HOUR,
+                scope: 'region',
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+    });
+
+    it('falls back to the domain as its namespace', () => {
+        expect(node({ domain: 'example.test' }).namespace).toBe('example.test');
+        expect(
+            node({ namespace: 'explicit', domain: 'example.test' }).namespace,
+        ).toBe('explicit');
+    });
+
+    it('boots a server whose domain carries a port, registering its jobs under the default namespace', async () => {
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const portDomain = await setupTestServer({
+            domain: 'localhost:4101',
+        } as IConfig);
+        try {
+            expect(portDomain.services.jobs.namespace).toBe('default');
+            const namespaceErrors = logged.mock.calls.filter(([message]) =>
+                String(message).startsWith('[jobs] namespace'),
+            );
+            expect(namespaceErrors).toHaveLength(1);
+        } finally {
+            await portDomain.shutdown();
+        }
+    });
+
+    // -- Region scope -------------------------------------------------
+
+    it('runs a region job once per region', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17101 });
+        const redisB = new MockRedis({ port: 17102 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.region', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs: 5_000,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        expect(
+            await redisA.pttl(`jobs:{${namespace}:job.region}:next`),
+        ).toBeGreaterThan(HOUR - 60_000);
+        // The region lock was released after the run.
+        expect(await redisA.exists(`jobs:{${namespace}:job.region}:lock`)).toBe(
+            0,
+        );
+    });
+
+    it('backs off a failing region job and keeps its state in Redis', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17103 });
+        const scheduler = node({ namespace, redis });
+        const seen: unknown[] = [];
+        let fail = true;
+        scheduler.register('job.regionFail', {
+            every: HOUR,
+            scope: 'region',
+            leaseMs: 5_000,
+            run: async ({ state, saveState }) => {
+                seen.push(state);
+                await saveState({ page: seen.length });
+                if (fail) throw new Error('nope');
+            },
+        });
+        const gate = `jobs:{${namespace}:job.regionFail}:next`;
+
+        await scheduler.poll();
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBe('1');
+        expect(await redis.pttl(gate)).toBeLessThanOrEqual(2_000);
+
+        fail = false;
+        await redis.del(gate);
+        await scheduler.poll();
+        expect(seen).toEqual([null, { page: 1 }]);
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBeNull();
+        expect(await redis.pttl(gate)).toBeGreaterThan(HOUR - 60_000);
+    });
+
+    it('skips region jobs quietly while Redis is unreachable', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17108 });
+        const scheduler = node({ namespace, redis });
+        const run = vi.fn(async () => {});
+        scheduler.register('job.noRedis', {
+            every: HOUR,
+            scope: 'region',
+            run,
+        });
+        vi.spyOn(redis, 'exists').mockRejectedValue(
+            new Error('connection refused'),
+        );
+        const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(errors).not.toHaveBeenCalled();
+    });
+
+    it('fences a region run that outlived its gate off its successor’s keys', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17105 });
+        const [slow, next] = [
+            node({ namespace, redis }),
+            node({ namespace, redis }),
+        ];
+        const keys = {
+            gate: `jobs:{${namespace}:job.fenced}:next`,
+            run: `jobs:{${namespace}:job.fenced}:run`,
+        };
+        let release!: () => void;
+        let slowStarted = false;
+        let slowSaveError: unknown = null;
+        // Ignores its signal and outlives both its 200 ms lock and 400 ms gate.
+        slow.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                slowStarted = true;
+                await new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+                try {
+                    await saveState({ by: 'slow' });
+                } catch (err) {
+                    slowSaveError = err;
+                }
+                throw new Error('slow run failed');
+            },
+        });
+        next.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                await saveState({ by: 'next' });
+            },
+        });
+
+        const slowRun = slow.poll();
+        await vi.waitFor(() => expect(slowStarted).toBe(true));
+        await sleep(450);
+        await next.poll();
+        const successorGate = await redis.get(keys.gate);
+        expect(successorGate).not.toBeNull();
+
+        release();
+        await slowRun;
+        expect(slowSaveError).toBeInstanceOf(JobLeaseLostError);
+        // Neither the slow run's state, its failure count, nor its cleanup
+        // touched what the successor holds.
+        expect(await redis.get(keys.gate)).toBe(successorGate);
+        expect(await redis.pttl(keys.gate)).toBeGreaterThan(0);
+        expect(await redis.hget(keys.run, 'state')).toBe('{"by":"next"}');
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+    it('runs a fleet job once per region, its lease capped at its cadence and nearPrimary ignored', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17111 });
+        const redisB = new MockRedis({ port: 17112 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const keys = keysOf(namespace, 'job.fleet');
+        const runs: string[] = [];
+        const lockTtls: number[] = [];
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        nodes.forEach((scheduler, i) => {
+            const redis = i < 2 ? redisA : redisB;
+            scheduler.register('job.fleet', {
+                every: 2_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                // Longer than `every`: refused for a region job, capped here.
+                leaseMs: HOUR,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    lockTtls.push(await redis.pttl(keys.lock));
+                    await sleep(20);
+                },
+            });
+        });
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        for (const ttl of lockTtls) {
+            expect(ttl).toBeGreaterThan(0);
+            expect(ttl).toBeLessThanOrEqual(2_000);
+        }
+        expect(await redisA.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(await redisB.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+    });
+});

src/backend/services/jobs/JobScheduler.ts

@@ -0,0 +1,528 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { withRedisLock } from '../../util/redisLock.js';
+import { resolveJobsNamespace } from '../../util/jobsNamespace.js';
+import { deliveryBackoffMs } from '../../util/retryBackoff.js';
+import { PuterService } from '../types';
+import {
+    JobLeaseLostError,
+    asError,
+    lostLease,
+    parseState,
+    runOutcome,
+    serializeState,
+    type RegisteredJob,
+    type RunLease,
+} from './jobRun.js';
+
+export { JobLeaseLostError } from './jobRun.js';
+
+/** How often each node looks for due jobs. */
+const JOB_POLL_MS = 30_000;
+/** Each poll waits up to this long first, so nodes don't all claim at once. */
+const JOB_POLL_JITTER_MS = 5_000;
+const JOB_DEFAULT_LEASE_MS = 15 * 60_000;
+/** Consecutive failures that raise `scheduled_job_failing`. */
+const JOB_FAILING_ALARM_AT = 3;
+const JOB_SHUTDOWN_WAIT_MS = 10_000;
+/**
+ * Job names: ASCII, at most 128, and without `:`, so a (namespace, name) pair
+ * maps to exactly one set of Redis keys (namespaces follow the same rule).
+ */
+const JOB_ID_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+// Region writes apply only while the gate still holds this run's token.
+const REGION_FENCED_WRITE = `
+if redis.call('get', KEYS[1]) ~= ARGV[1] then return -1 end
+local op = ARGV[2]
+if op == 'hset' then return redis.call('hset', KEYS[2], ARGV[3], ARGV[4]) end
+if op == 'hdel' then return redis.call('hdel', KEYS[2], ARGV[3]) end
+if op == 'hincrby' then return redis.call('hincrby', KEYS[2], ARGV[3], 1) end
+if op == 'pexpire' then return redis.call('pexpire', KEYS[1], ARGV[3]) end
+if op == 'del' then return redis.call('del', KEYS[1]) end
+return -1
+`;
+
+/**
+ * `region` runs once per cadence per region. `fleet` means once across every
+ * region where `services.jobs` supports it; this default runs it per region.
+ */
+export type JobScope = 'fleet' | 'region';
+
+export interface JobRunContext {
+    /** Aborts once the run may have lost the job. Check it before each batch. */
+    signal: AbortSignal;
+    /** What an earlier run saved, or null. */
+    state: unknown;
+    /**
+     * Persist JSON-serializable `state` for later runs. Throws once the run
+     * lost the job.
+     */
+    saveState: (state: unknown) => Promise<void>;
+}
+
+export interface JobDefinition {
+    /** Cadence in ms. */
+    every: number;
+    scope: JobScope;
+    /**
+     * Prefer the region nearest the primary database, for a `fleet` job. The
+     * default scheduler ignores it.
+     */
+    nearPrimary?: boolean;
+    /**
+     * How long a run holds the job, in total on the region path (at most
+     * `every`). Default 15 min, capped at `every` for `region`; a `fleet` job
+     * run per region is capped at `every` too.
+     */
+    leaseMs?: number;
+    run: (ctx: JobRunContext) => Promise<void>;
+}
+
+/** The `afterId` cursor a keyset-walking job saved in its state, or 0. */
+export const savedAfterId = (state: unknown): number => {
+    const afterId = (state as { afterId?: unknown } | null)?.afterId;
+    return typeof afterId === 'number' &&
+        Number.isSafeInteger(afterId) &&
+        afterId > 0
+        ? afterId
+        : 0;
+};
+
+interface RegionKeys {
+    lock: string;
+    gate: string;
+    run: string;
+}
+
+/**
+ * Runs our own recurring backend work once per cadence per region, under a
+ * Redis lock. `fleet` jobs take the same path here; an extension registered as
+ * `services.jobs` can run them fleet-wide by overriding `pollFleet`. Nothing
+ * polls until a job is registered.
+ */
+export class JobScheduler extends PuterService {
+    #jobs = new Map<string, RegisteredJob>();
+    #running = new Map<string, Promise<void>>();
+    #leases = new Set<RunLease>();
+    #started = false;
+    #stopping = false;
+    /** Polling stopped; runs already going keep their leases. */
+    #draining = false;
+    #timer: ReturnType<typeof setInterval> | null = null;
+    #pollDelays = new Set<ReturnType<typeof setTimeout>>();
+    #warnedOff = false;
+    #namespace: string | null = null;
+
+    /** Keys this deployment's jobs, so deployments sharing Redis stay apart. */
+    get namespace(): string {
+        return (this.#namespace ??= resolveJobsNamespace(this.config));
+    }
+
+    register(name: string, definition: JobDefinition): void {
+        if (!JOB_ID_PATTERN.test(name)) {
+            throw new Error(`invalid job name: ${name}`);
+        }
+        if (this.#jobs.has(name)) {
+            throw new Error(`job already registered: ${name}`);
+        }
+        const { every, scope } = definition;
+        if (!Number.isSafeInteger(every) || every <= 0) {
+            throw new Error(`job ${name}: every must be a positive integer`);
+        }
+        if (scope !== 'fleet' && scope !== 'region') {
+            throw new Error(`job ${name}: unknown scope`);
+        }
+        const leaseMs =
+            definition.leaseMs ??
+            (scope === 'region'
+                ? Math.min(JOB_DEFAULT_LEASE_MS, every)
+                : JOB_DEFAULT_LEASE_MS);
+        if (!Number.isSafeInteger(leaseMs) || leaseMs <= 0) {
+            throw new Error(`job ${name}: leaseMs must be a positive integer`);
+        }
+        // The gate that fences a region run lasts `every`.
+        if (scope === 'region' && leaseMs > every) {
+            throw new Error(
+                `job ${name}: a region job's leaseMs exceeds every`,
+            );
+        }
+        this.#jobs.set(name, {
+            every,
+            scope,
+            nearPrimary: definition.nearPrimary ?? false,
+            leaseMs,
+            run: definition.run,
+        });
+        if (this.#started) this.#arm();
+    }
+
+    override onServerStart(): void {
+        this.#started = true;
+        if (this.#jobs.size > 0) this.#arm();
+    }
+
+    /** Claim nothing new while draining; running jobs carry on until prepare. */
+    override onServerDrainStart(): void {
+        this.#stopPolling();
+    }
+
+    override onServerPrepareShutdown(): void {
+        this.#stopping = true;
+        this.#stopPolling();
+        for (const lease of [...this.#leases]) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+            lease.close();
+        }
+    }
+
+    #stopPolling(): void {
+        this.#draining = true;
+        if (this.#timer) clearInterval(this.#timer);
+        this.#timer = null;
+        for (const delay of this.#pollDelays) clearTimeout(delay);
+        this.#pollDelays.clear();
+    }
+
+    /**
+     * Give aborted runs a moment to record themselves before the database
+     * closes.
+     */
+    override async onServerShutdown(): Promise<void> {
+        if (this.#running.size === 0) return;
+        let wait: ReturnType<typeof setTimeout> | undefined;
+        await Promise.race([
+            Promise.allSettled(this.#running.values()),
+            new Promise((resolve) => {
+                wait = setTimeout(resolve, JOB_SHUTDOWN_WAIT_MS);
+            }),
+        ]);
+        clearTimeout(wait);
+    }
+
+    /**
+     * Start every registered job that is due and not already running here.
+     * Resolves once the runs it started have finished.
+     */
+    async poll(): Promise<void> {
+        if (this.#stopping || this.#draining) return;
+        const fleet: string[] = [];
+        const region: Promise<void>[] = [];
+        for (const [name, job] of this.#jobs) {
+            if (this.#running.has(name)) continue;
+            if (job.scope === 'fleet') {
+                fleet.push(name);
+                continue;
+            }
+            region.push(this.track(name, () => this.#tryRegion(name, job)));
+        }
+        await Promise.all([this.pollFleet(fleet), ...region]);
+    }
+
+    #arm(): void {
+        if (this.#timer || this.#stopping || this.#draining) return;
+        if (this.config.jobs?.enabled === false) {
+            // Nothing alarms on a job that never runs, so say it here.
+            if (!this.#warnedOff) {
+                console.warn(
+                    '[jobs] polling is off on this node (jobs.enabled: false); other nodes must run the jobs',
+                );
+            }
+            this.#warnedOff = true;
+            return;
+        }
+        const tick = () => {
+            const delay = setTimeout(() => {
+                this.#pollDelays.delete(delay);
+                void this.#pollSafely();
+            }, Math.random() * JOB_POLL_JITTER_MS);
+            delay.unref?.();
+            this.#pollDelays.add(delay);
+        };
+        const timer = setInterval(tick, JOB_POLL_MS);
+        timer.unref?.();
+        this.#timer = timer;
+        tick();
+    }
+
+    async #pollSafely(): Promise<void> {
+        try {
+            await this.poll();
+        } catch (err) {
+            console.error('[jobs] poll failed', err);
+        }
+    }
+
+    // -- Region scope -------------------------------------------------
+
+    async #tryRegion(name: string, job: RegisteredJob): Promise<void> {
+        const redis = this.clients.redis;
+        const keys = regionKeys(this.namespace, name);
+        let gated: number;
+        try {
+            gated = await redis.exists(keys.gate);
+        } catch {
+            // Redis is unreachable; the lock below would be busy anyway.
+            return;
+        }
+        if (gated > 0) return;
+        // Before the lock, so acquiring it counts against `leaseMs`.
+        const startedAt = performance.now();
+        await withRedisLock(
+            redis,
+            keys.lock,
+            async () => {
+                const token = randomUUID();
+                const gate = await redis.set(
+                    keys.gate,
+                    token,
+                    'PX',
+                    job.every,
+                    'NX',
+                );
+                if (gate !== 'OK') return;
+                await this.#runRegion(name, job, keys, token, startedAt);
+            },
+            {
+                ttlMs: job.leaseMs,
+                onUnavailable: 'busy',
+                onBusy: (): void => undefined,
+            },
+        );
+    }
+
+    /**
+     * One run under the region lock. The lock doesn't renew, so the signal
+     * aborts once it lapses, and every write is fenced on the gate's token.
+     */
+    async #runRegion(
+        name: string,
+        job: RegisteredJob,
+        keys: RegionKeys,
+        token: string,
+        startedAt: number,
+    ): Promise<void> {
+        const lease = this.#regionLease(job.leaseMs, startedAt);
+        const { signal } = lease.controller;
+        const write = (op: string, ...args: (string | number)[]) =>
+            this.#regionWrite(keys, token, op, ...args);
+        let error: unknown = null;
+        try {
+            const saved = await this.clients.redis.hget(keys.run, 'state');
+            if (await lease.ready()) {
+                await job.run({
+                    signal,
+                    state: parseState(saved),
+                    saveState: async (state) => {
+                        const json = serializeState(state);
+                        if ((await write('hset', 'state', json)) < 0) {
+                            lostLease(lease, name);
+                        }
+                    },
+                });
+            }
+        } catch (err) {
+            error = err;
+        } finally {
+            lease.close();
+        }
+
+        const outcome = runOutcome(signal, error);
+        if (outcome === 'success') {
+            await write('hdel', 'failures');
+        } else if (outcome === 'failure') {
+            const failures = await write('hincrby', 'failures');
+            if (failures < 0) return;
+            await write(
+                'pexpire',
+                Math.min(job.every, deliveryBackoffMs(failures)),
+            );
+            this.reportFailure(name, failures, error);
+        } else {
+            await write('del');
+        }
+    }
+
+    /**
+     * The lock lapses `leaseMs` after `from`; the run needs half of it left to
+     * start.
+     */
+    #regionLease(leaseMs: number, from: number): RunLease {
+        const controller = new AbortController();
+        const deadline = from + leaseMs;
+        const abort = (reason: string) => {
+            if (!controller.signal.aborted) {
+                controller.abort(new JobLeaseLostError(reason));
+            }
+        };
+        const lapse = setTimeout(
+            () => abort('region lock lapsed'),
+            Math.max(0, deadline - performance.now()),
+        );
+        const lease: RunLease = {
+            controller,
+            ready: async () => {
+                if (
+                    !controller.signal.aborted &&
+                    deadline - performance.now() >= leaseMs / 2
+                ) {
+                    return true;
+                }
+                abort('too little of the lock left to start');
+                return false;
+            },
+            close: () => {
+                clearTimeout(lapse);
+                this.closeLease(lease);
+            },
+        };
+        this.openLease(lease);
+        return lease;
+    }
+
+    /**
+     * Run a write to the region keys if the gate still holds `token`; -1 if
+     * not.
+     */
+    async #regionWrite(
+        keys: RegionKeys,
+        token: string,
+        op: string,
+        ...args: (string | number)[]
+    ): Promise<number> {
+        const result = await this.clients.redis.eval(
+            REGION_FENCED_WRITE,
+            2,
+            keys.gate,
+            keys.run,
+            token,
+            op,
+            ...args,
+        );
+        return Number(result);
+    }
+
+    // -- Hooks, also for extensions that replace `services.jobs` -------
+
+    /**
+     * Run `fleet` jobs registered here and not running in this process. The
+     * default holds each per region, its lease capped at `every`.
+     */
+    protected async pollFleet(names: string[]): Promise<void> {
+        await Promise.all(
+            names.map((name) => {
+                const job = this.definitionOf(name);
+                return this.track(name, () =>
+                    this.#tryRegion(name, {
+                        ...job,
+                        leaseMs: Math.min(job.leaseMs, job.every),
+                    }),
+                );
+            }),
+        );
+    }
+
+    /** Runs `work` as `name`'s only run in this process. */
+    protected track(name: string, work: () => Promise<void>): Promise<void> {
+        const run = (async () => {
+            try {
+                await work();
+            } catch (err) {
+                console.error(`[jobs] ${name} errored`, err);
+            } finally {
+                this.#running.delete(name);
+            }
+        })();
+        this.#running.set(name, run);
+        return run;
+    }
+
+    protected isRunning(name: string): boolean {
+        return this.#running.has(name);
+    }
+
+    protected definitionOf(name: string): RegisteredJob {
+        const job = this.#jobs.get(name);
+        if (!job) throw new Error(`job not registered: ${name}`);
+        return job;
+    }
+
+    /** Who runs a job here: region, server and process. */
+    protected owner(): string {
+        const region = this.services.broadcast.regionId;
+        const server = this.config.serverId ?? 'local';
+        return `${region}:${server}:${process.pid}`.slice(0, 128);
+    }
+
+    /**
+     * Track `lease` for shutdown; one opened while shutting down is aborted at
+     * once.
+     */
+    protected openLease(lease: RunLease): void {
+        this.#leases.add(lease);
+        if (this.#stopping) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+        }
+    }
+
+    /** Stop tracking `lease`; its `close` calls this. */
+    protected closeLease(lease: RunLease): void {
+        this.#leases.delete(lease);
+    }
+
+    protected reportFailure(
+        name: string,
+        failures: number,
+        error: unknown,
+    ): void {
+        console.error(`[jobs] ${name} failed (${failures} in a row)`, error);
+        if (failures < JOB_FAILING_ALARM_AT) return;
+        this.clients.alarm.create(
+            `scheduled_job_failing:${this.namespace}:${name}`,
+            `System job ${name} keeps failing`,
+            {
+                namespace: this.namespace,
+                name,
+                failures,
+                error: asError(error),
+            },
+            'warning',
+            { dedup: true },
+        );
+    }
+}
+
+/**
+ * One hash slot per job. Neither part may contain `:` (`JOB_ID_PATTERN`), so no
+ * two (namespace, name) pairs share keys.
+ */
+const regionKeys = (namespace: string, name: string): RegionKeys => {
+    const tag = `{${namespace}:${name}}`;
+    return {
+        lock: `jobs:${tag}:lock`,
+        gate: `jobs:${tag}:next`,
+        run: `jobs:${tag}:run`,
+    };
+};

src/backend/services/jobs/jobRun.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// What one job run needs, whichever scheduler holds the job. Also used by
+// extensions that replace `services.jobs`.
+
+import type { JobDefinition } from './JobScheduler.js';
+
+export const JOB_STATE_MAX_BYTES = 64 * 1024;
+export const JOB_ERROR_MAX_CHARS = 1024;
+
+/** The run no longer holds its job; another node may already be running it. */
+export class JobLeaseLostError extends Error {}
+
+export type RegisteredJob = Required<JobDefinition>;
+
+export type RunOutcome = 'success' | 'failure' | 'interrupted';
+
+/** What a run holds the job by. Its deadline is measured from before the claim. */
+export interface RunLease {
+    controller: AbortController;
+    /**
+     * Renews if little lease is left; false (and aborted) if the run can't
+     * start.
+     */
+    ready: () => Promise<boolean>;
+    close: () => void;
+}
+
+export const lostLease = (lease: RunLease, name: string): never => {
+    const lost = new JobLeaseLostError(`job ${name} lost its lease`);
+    lease.controller.abort(lost);
+    throw lost;
+};
+
+/** A run cut short by its signal is interrupted, whether or not it threw. */
+export const runOutcome = (signal: AbortSignal, error: unknown): RunOutcome => {
+    if (signal.aborted) return 'interrupted';
+    return error === null ? 'success' : 'failure';
+};
+
+export const parseState = (raw: string | null): unknown => {
+    if (raw === null) return null;
+    try {
+        return JSON.parse(raw);
+    } catch {
+        return null;
+    }
+};
+
+export const serializeState = (state: unknown): string => {
+    const json = JSON.stringify(state ?? null);
+    if (Buffer.byteLength(json) > JOB_STATE_MAX_BYTES) {
+        throw new Error(`job state is over ${JOB_STATE_MAX_BYTES} bytes`);
+    }
+    return json;
+};
+
+export const asError = (error: unknown): Error =>
+    error instanceof Error ? error : new Error(String(error));
+
+export const errorText = (error: unknown): string =>
+    (error instanceof Error ? error.message : String(error)).slice(
+        0,
+        JOB_ERROR_MAX_CHARS,
+    );

src/backend/services/jobs/jobsTestUtil.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type { JobRunContext } from './JobScheduler.js';
+
+/**
+ * What a system job's `run` receives, for calling it directly: `saved` records
+ * each `saveState`, and `abort` aborts the signal.
+ */
+export const testJobContext = (
+    state: unknown = null,
+): JobRunContext & { saved: unknown[]; abort: () => void } => {
+    const controller = new AbortController();
+    const saved: unknown[] = [];
+    return {
+        signal: controller.signal,
+        state,
+        saveState: async (next) => {
+            saved.push(next);
+        },
+        saved,
+        abort: () => controller.abort(),
+    };
+};

src/backend/services/jobs/systemJobs.test.ts

@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The backend's own periodic jobs on the default scheduler: what registers,
+ * that each runs once across nodes sharing Redis, and that a single self-hosted
+ * node (sqlite, in-process Redis) runs every one of them.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobScheduler,
+    type JobDefinition,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const MINUTE = 60_000;
+const HOUR = 60 * MINUTE;
+
+const EXPECTED_JOBS = {
+    'auth.sessions.purge': {
+        every: 7 * 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 30 * MINUTE,
+    },
+    'kvShare.handles.purgeRevoked': {
+        every: 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'notifications.retention': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'metering.globalRateCheck': {
+        every: 25 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 5 * MINUTE,
+    },
+    'events.subscriptions.expirySweep': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'events.subscriptions.creditSweep': {
+        every: 15 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 10 * MINUTE,
+    },
+};
+const JOB_NAMES = Object.keys(EXPECTED_JOBS);
+
+/** A timer armed while the server booted, and the frame that armed it. */
+interface ArmedTimer {
+    kind: 'interval' | 'timeout';
+    ms: number | undefined;
+    caller: string;
+}
+
+let server: PuterServer;
+const registered = new Map<string, JobDefinition>();
+const armed: ArmedTimer[] = [];
+
+/** The first stack frame outside this file and node_modules. */
+const callerOf = (stack: string | undefined): string =>
+    (stack ?? '')
+        .split('\n')
+        .slice(1)
+        .find(
+            (line) =>
+                !line.includes('systemJobs.test') &&
+                !line.includes('node_modules'),
+        ) ?? '';
+
+beforeAll(async () => {
+    const register = vi.spyOn(JobScheduler.prototype, 'register');
+    const realInterval = globalThis.setInterval;
+    const realTimeout = globalThis.setTimeout;
+    const intervals = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
+        ...args: Parameters<typeof setInterval>
+    ) => {
+        armed.push({
+            kind: 'interval',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realInterval(...args);
+    }) as never);
+    const timeouts = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+        ...args: Parameters<typeof setTimeout>
+    ) => {
+        armed.push({
+            kind: 'timeout',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realTimeout(...args);
+    }) as never);
+    try {
+        server = await setupTestServer({
+            events: { enabled: true },
+            notificationRetentionDays: 14,
+            maxGlobalUsagePerMinute: 1e15,
+            // A self-hosted node: nothing turns its jobs off.
+            jobs: { enabled: true },
+        } as IConfig);
+    } finally {
+        intervals.mockRestore();
+        timeouts.mockRestore();
+    }
+    for (const [name, definition] of register.mock.calls)
+        registered.set(name, definition);
+    register.mockRestore();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+const keysOf = (namespace: string, name: string) => ({
+    lock: `jobs:{${namespace}:${name}}:lock`,
+    gate: `jobs:{${namespace}:${name}}:next`,
+    run: `jobs:{${namespace}:${name}}:run`,
+});
+
+/** A node of its own in `region`, on the test server's Redis. */
+const node = (namespace: string, region: string): JobScheduler =>
+    new JobScheduler(
+        {
+            domain: 'puter.test',
+            serverId: `node-${region}`,
+            jobs: { namespace },
+        } as IConfig,
+        server.clients as SchedulerArgs[1],
+        server.stores as SchedulerArgs[2],
+        {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3],
+    );
+
+describe('system jobs', () => {
+    it('registers every moved job with its real cadence, scope and lease', () => {
+        expect(
+            Object.fromEntries(
+                [...registered].map(([name, job]) => [
+                    name,
+                    {
+                        every: job.every,
+                        scope: job.scope,
+                        nearPrimary: job.nearPrimary ?? false,
+                        leaseMs: job.leaseMs,
+                    },
+                ]),
+            ),
+        ).toEqual(EXPECTED_JOBS);
+    });
+
+    it('leaves no per-node timer behind for a moved job', () => {
+        const armedFrom = (file: string) =>
+            armed
+                .filter(({ caller }) => caller.includes(file))
+                .map(({ kind, ms }) => ({ kind, ms }));
+
+        expect(armedFrom('services/auth/AuthService')).toEqual([]);
+        expect(armedFrom('services/notification/NotificationService')).toEqual(
+            [],
+        );
+        // Node-local: the usage buffer flush.
+        expect(armedFrom('services/metering/MeteringService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+        // Node-local: the share-job runner (DB claims per job), and its kick.
+        expect(armedFrom('services/kvShare/KvShareService')).toEqual([
+            { kind: 'timeout', ms: expect.any(Number) },
+            { kind: 'interval', ms: MINUTE },
+        ]);
+        // Node-local: the region's pending-delivery sweep.
+        expect(armedFrom('services/events/EventsService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+    });
+
+    it.each(JOB_NAMES)(
+        'runs %s once across three nodes sharing Redis',
+        async (name) => {
+            const namespace = `ns-${randomUUID()}`;
+            const nodes = ['n1', 'n2', 'n3'].map((id) => node(namespace, id));
+            const job = registered.get(name)!;
+            let runs = 0;
+            for (const scheduler of nodes)
+                scheduler.register(name, {
+                    ...job,
+                    run: async (ctx: JobRunContext) => {
+                        runs += 1;
+                        await job.run(ctx);
+                    },
+                });
+
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+
+            expect(runs).toBe(1);
+            const redis = server.clients.redis;
+            const keys = keysOf(namespace, name);
+            // A failure would have cut the gate to a 2 s backoff.
+            expect(await redis.pttl(keys.gate)).toBeGreaterThan(
+                job.every - MINUTE,
+            );
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        },
+    );
+
+    it('runs every job on a single self-hosted node, without the old per-job gates', async () => {
+        const jobs = server.services.jobs;
+        const redis = server.clients.redis;
+
+        await jobs.poll();
+
+        // The node's own poll timer may have started some of them first.
+        await vi.waitFor(
+            async () => {
+                for (const name of JOB_NAMES) {
+                    const keys = keysOf(jobs.namespace, name);
+                    expect(await redis.exists(keys.gate), name).toBe(1);
+                    // Released once the run is over.
+                    expect(await redis.exists(keys.lock), name).toBe(0);
+                    expect(await redis.hget(keys.run, 'failures'), name).toBe(
+                        null,
+                    );
+                }
+            },
+            { timeout: 10_000, interval: 100 },
+        );
+        await expect(
+            redis.exists('sessions:purge:gate', 'kvShare:purge:gate'),
+        ).resolves.toBe(0);
+    });
+});

src/backend/services/kvShare/KvShareService.ts

@@ -371,11 +371,9 @@ const SHARE_CHANGE_LOCK_RETRY_MS = 50;
 
 /** Retired handles are kept this long for their owner's listing, then deleted. */
 export const KV_SHARE_REVOKED_RETENTION_SECONDS = 30 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per day.
-const PURGE_CHECK_MS = 60 * 60 * 1000;
-const PURGE_INITIAL_DELAY_MS = 5 * 60 * 1000;
-const PURGE_GATE_KEY = 'kvShare:purge:gate';
-const PURGE_GATE_SECONDS = 24 * 60 * 60;
+const PURGE_JOB = 'kvShare.handles.purgeRevoked';
+const PURGE_EVERY_MS = 24 * 60 * 60 * 1000;
+const PURGE_LEASE_MS = 15 * 60 * 1000;
 // Rows per delete, with a pause between, so replicas keep up.
 const PURGE_BATCH = 500;
 const PURGE_PAUSE_MS = 50;
@@ -536,7 +534,17 @@ export class KvShareService extends PuterService {
     #jobCursor: string | undefined;
 
     override onServerStart(): void {
-        this.#armPurge();
+        this.services.jobs.register(PURGE_JOB, {
+            every: PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: PURGE_LEASE_MS,
+            run: async ({ signal }) => {
+                const purged = await this.purgeRevokedHandles(signal);
+                if (purged)
+                    console.log(`[kvShare] purged ${purged} retired handles`);
+            },
+        });
         this.#armJobs();
 
         // A check at one level is answered by a grant at any level above it.
@@ -607,51 +615,24 @@ export class KvShareService extends PuterService {
     // -- Retention purge ---------------------------------------------
 
     /**
-     * Delete handles retired longer ago than the retention, in batches. Returns
-     * the rows deleted, or null when another node holds today's gate. A missing
-     * row reads like a retired one everywhere a handle is resolved.
+     * Delete handles retired longer ago than the retention, in batches, until
+     * none are left or `signal` aborts. Returns the rows deleted. A missing row
+     * reads like a retired one everywhere a handle is resolved.
      */
-    async purgeRevokedHandles(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeRevokedHandles(signal: AbortSignal): Promise<number> {
         const cutoff =
             Math.floor(Date.now() / 1000) - KV_SHARE_REVOKED_RETENTION_SECONDS;
         let purged = 0;
-        let finished = false;
-        try {
-            for (let i = 0; i < PURGE_MAX_BATCHES; i++) {
-                if (this.#stopping) return purged;
-                const batch =
-                    await this.stores.kvShareHandle.deleteRevokedBefore(
-                        cutoff,
-                        PURGE_BATCH,
-                    );
-                purged += batch;
-                if (batch < PURGE_BATCH) break;
-                await new Promise((resolve) =>
-                    setTimeout(resolve, PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A run cut short frees the gate, so another node picks it up on
-            // its next check instead of tomorrow.
-            if (!finished) {
-                try {
-                    await this.clients.redis.del(PURGE_GATE_KEY);
-                } catch {
-                    // The gate expires on its own.
-                }
-            }
+        for (let i = 0; i < PURGE_MAX_BATCHES && !signal.aborted; i++) {
+            const batch = await this.stores.kvShareHandle.deleteRevokedBefore(
+                cutoff,
+                PURGE_BATCH,
+            );
+            purged += batch;
+            if (batch < PURGE_BATCH) break;
+            await new Promise((resolve) => setTimeout(resolve, PURGE_PAUSE_MS));
         }
+        return purged;
     }
 
     /**
@@ -1047,27 +1028,6 @@ export class KvShareService extends PuterService {
         this.#timers.push(kick, timer);
     }
 
-    #armPurge(): void {
-        const run = async () => {
-            try {
-                const purged = await this.purgeRevokedHandles();
-                if (purged)
-                    console.log(`[kvShare] purged ${purged} retired handles`);
-            } catch (err) {
-                console.warn('[kvShare] retired-handle purge failed', err);
-            }
-        };
-        // Jittered so a deploy does not have every node check at once.
-        const kick = setTimeout(
-            () => void run(),
-            PURGE_INITIAL_DELAY_MS * (0.5 + Math.random()),
-        );
-        kick.unref?.();
-        const timer = setInterval(() => void run(), PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#timers.push(kick, timer);
-    }
-
     /** Whether the level a mint asks for is available on this server. */
     #assertAccessEnabled(access: KvShareAccess): void {
         if (access === 'watch') {
@@ -1745,11 +1705,14 @@ export class KvShareService extends PuterService {
      * The other party of a share, for a notification one party sends through
      * its handle from the app whose data it is. Every check runs before the
      * pair budget is spent. A handle that is unknown, retired, not the
-     * caller's, or seen from another app reads as absent.
+     * caller's, or seen from another app reads as absent. `spendBudget: false`
+     * checks without spending, for a send that is only being scheduled. Called
+     * by extensions.
      */
     async resolveNotifyRecipient(
         actor: Actor,
         handle: unknown,
+        { spendBudget = true }: { spendBudget?: boolean } = {},
     ): Promise<KvShareNotifyTarget> {
         const named = typeof handle === 'string' ? handle : '';
         const senderId = actor.user?.id;
@@ -1782,7 +1745,8 @@ export class KvShareService extends PuterService {
         if (!checked?.holds) throw unknownKvShareHandle(named);
         if (checked.refuses) throw recipientNotAcceptingShares();
 
-        await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
+        if (spendBudget)
+            await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
         return {
             recipientUserId: recipientId,
             appUid: share.appUid,

src/backend/services/metering/MeteringService.ts

@@ -111,6 +111,11 @@ type UsageWriteResult = {
     detail?: FlatUsageDetail;
 };
 
+/** Compares fleet-wide spend against `maxGlobalUsagePerMinute`. */
+const GLOBAL_RATE_CHECK_JOB = 'metering.globalRateCheck';
+const GLOBAL_RATE_CHECK_EVERY_MS = 25 * 60 * 1000;
+const GLOBAL_RATE_CHECK_LEASE_MS = 5 * 60 * 1000;
+
 // -- Helpers ----------------------------------------------------------
 
 /**
@@ -284,7 +289,6 @@ export class MeteringService extends PuterService {
      */
     static APP_TOTALS_LIST_LIMIT = 4_000;
 
-    private rateCheckTimer: ReturnType<typeof setInterval> | null = null;
     private usageBufferTimer: ReturnType<typeof setInterval> | null = null;
     private extraPolicies: SubscriptionPolicy[] = [];
     private subscriptionResolvers: SubscriptionResolver[] = [];
@@ -415,15 +419,15 @@ export class MeteringService extends PuterService {
             },
         );
 
-        this.rateCheckTimer = setInterval(
-            () => {
-                this.checkRateOfChange().catch((e) => {
-                    console.error('[metering] rate-of-change check failed', e);
-                });
-            },
-            1000 * 60 * 25,
-        );
-        this.rateCheckTimer.unref?.();
+        const maxPerMinute = this.config.maxGlobalUsagePerMinute;
+        if (maxPerMinute && maxPerMinute > 0) {
+            this.services.jobs.register(GLOBAL_RATE_CHECK_JOB, {
+                every: GLOBAL_RATE_CHECK_EVERY_MS,
+                scope: 'fleet',
+                leaseMs: GLOBAL_RATE_CHECK_LEASE_MS,
+                run: () => this.checkRateOfChange(),
+            });
+        }
 
         const flushInterval =
             this.config.meteringUsageBufferFlushMs &&
@@ -454,10 +458,6 @@ export class MeteringService extends PuterService {
     }
 
     override async onServerShutdown(): Promise<void> {
-        if (this.rateCheckTimer) {
-            clearInterval(this.rateCheckTimer);
-            this.rateCheckTimer = null;
-        }
         if (this.usageBufferTimer) {
             clearInterval(this.usageBufferTimer);
             this.usageBufferTimer = null;
@@ -2948,6 +2948,8 @@ export class MeteringService extends PuterService {
             timestamp: number;
         } | null;
 
+        // Several schedulers can share the baseline (one per region without a
+        // fleet scheduler, or old nodes mid-rollout); keep windows wide.
         if (lastChange && now - lastChange.timestamp <= 14 * 60 * 1000) return;
 
         const globalUsage = await this.getGlobalUsage();

src/backend/services/notification/NotificationService.test.ts

@@ -395,6 +395,36 @@ describe('NotificationService.notifyUpdate', () => {
         written.stop();
     });
 
+    it('sends the desktop rewrite before reading the row back, so a failed read still updates it', async () => {
+        const user = await makeUser();
+        const uid = await notifications.notify(
+            [user.id],
+            { title: 'one share' },
+            { type: 'share.received' },
+        );
+        const pushed = collect('outer.gui.notif.message');
+        const read = vi
+            .spyOn(server.stores.notification, 'getByUid')
+            .mockRejectedValueOnce(new Error('replica down'));
+        try {
+            await expect(
+                notifications.notifyUpdate(
+                    uid,
+                    user.id,
+                    { title: 'two shares' },
+                    { type: 'share.received' },
+                ),
+            ).rejects.toThrow('replica down');
+            expect(pushed.seen).toHaveLength(1);
+            expect((pushed.seen[0] as NotifPush).response.notification).toEqual(
+                { title: 'two shares', type: 'share.received' },
+            );
+        } finally {
+            read.mockRestore();
+            pushed.stop();
+        }
+    });
+
     it('refuses to fold a type that does not group', async () => {
         const user = await makeUser();
         await expect(
@@ -882,17 +912,42 @@ describe('NotificationService.sweepExpired', () => {
     });
 
     it('stops at the pass cap and leaves the rest for the next sweep', async () => {
+        // A backlog that always fills the batch: the cap, not a short batch,
+        // is what ends the call. Seeding 500k rows would take minutes.
+        const deletes = vi
+            .spyOn(server.stores.notification, 'deleteCreatedBefore')
+            .mockResolvedValue(500);
+        try {
+            expect(await notifications.sweepExpired()).toBe(500_000);
+            expect(deletes).toHaveBeenCalledTimes(1000);
+        } finally {
+            deletes.mockRestore();
+        }
+    });
+
+    it('stops between batches once its signal aborts', async () => {
         await notifications.sweepExpired();
         const user = await makeUser();
-        // 50 passes * 500/batch = 25,000 — a backlog past that so the cap,
-        // not a short batch, is what ends the first call.
-        await seedExpired(user.id, 25_050);
-
-        expect(await notifications.sweepExpired()).toBe(25_000);
-        expect(
-            await server.stores.notification.listByUserId(user.id),
-        ).toHaveLength(50);
-        expect(await notifications.sweepExpired()).toBe(50);
+        await seedExpired(user.id, 1200);
+        const controller = new AbortController();
+        const deleteBatch = server.stores.notification.deleteCreatedBefore.bind(
+            server.stores.notification,
+        );
+        const spy = vi
+            .spyOn(server.stores.notification, 'deleteCreatedBefore')
+            .mockImplementation(async (days: number, limit: number) => {
+                const removed = await deleteBatch(days, limit);
+                controller.abort();
+                return removed;
+            });
+        try {
+            expect(await notifications.sweepExpired(controller.signal)).toBe(
+                500,
+            );
+        } finally {
+            spy.mockRestore();
+        }
+        expect(await notifications.sweepExpired()).toBe(700);
     });
 
     it('sweeps nothing when no retention is configured', async () => {

src/backend/services/notification/NotificationService.ts

@@ -48,12 +48,16 @@ export type {
 } from './notificationTypes.js';
 export { canViewNotification } from './notificationAudience.js';
 
-/** How often the retention sweep runs. */
-const RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+const RETENTION_SWEEP_JOB = 'notifications.retention';
+const RETENTION_SWEEP_EVERY_MS = 60 * 60 * 1000;
+const RETENTION_SWEEP_LEASE_MS = 15 * 60 * 1000;
 /** Rows one delete takes. Small enough not to hold a lock anyone waits on. */
 const RETENTION_BATCH_SIZE = 500;
-/** Batches one sweep takes, so a large backlog drains over several passes. */
-const RETENTION_MAX_BATCHES = 50;
+/**
+ * Batches one sweep takes at most. It runs once for the fleet, so this is the
+ * fleet's hourly drain; the job's signal stops it sooner.
+ */
+const RETENTION_MAX_BATCHES = 1000;
 /** Unread notifications one reconnect replays. */
 const UNREAD_REPLAY_LIMIT = 200;
 
@@ -69,11 +73,20 @@ const UNREAD_REPLAY_LIMIT = 200;
 export class NotificationService extends PuterService {
     /** User.id → debounce timeout */
     #connectTimeouts = new Map<number, ReturnType<typeof setTimeout>>();
-    #retentionSweep: ReturnType<typeof setInterval> | null = null;
     #adapter: NotificationSocketAdapter | null = null;
 
     override onServerStart(): void {
-        this.#armRetentionSweep();
+        if (this.#retentionDays() > 0) {
+            this.services.jobs.register(RETENTION_SWEEP_JOB, {
+                every: RETENTION_SWEEP_EVERY_MS,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: RETENTION_SWEEP_LEASE_MS,
+                run: async ({ signal }) => {
+                    await this.sweepExpired(signal);
+                },
+            });
+        }
         this.#socket().attach();
 
         // When a user opens the GUI, send their pending unreads.
@@ -103,11 +116,6 @@ export class NotificationService extends PuterService {
         );
     }
 
-    override onServerPrepareShutdown(): void {
-        if (this.#retentionSweep) clearInterval(this.#retentionSweep);
-        this.#retentionSweep = null;
-    }
-
     // -- Public API --------------------------------------------------
 
     /**
@@ -176,25 +184,17 @@ export class NotificationService extends PuterService {
                 }
 
                 if (!opts.silent) {
-                    // Listeners get their own copy; the socket sends `payload`.
-                    this.clients.event.emit(
-                        'notif.written',
+                    await this.emitWritten(
                         {
                             userId,
                             uid,
                             type: registered.type,
                             audience: registered.audience,
                             appUid,
-                            value: structuredClone(payload),
-                            sendId: uuidv4(),
+                            value: payload,
                         },
-                        {},
+                        { sendId: uuidv4() },
                     );
-                    await this.#push(userId, uid, payload, {
-                        type: registered.type,
-                        audience: registered.audience,
-                        appUid,
-                    });
                 }
                 return uid;
             }),
@@ -241,50 +241,107 @@ export class NotificationService extends PuterService {
         if (!updated) return false;
         if (opts.silent) return true;
 
+        // Before the read, so the desktop copy updates even if it fails.
         const desktop = inAccountMailbox(registered.type);
-        if (desktop) {
-            // Same uid as the original: the client replaces what it is already
-            // showing rather than stacking another copy. It goes straight to
-            // the wire rather than through dispatch — the row was already
-            // published, and republishing it under its own id is a duplicate
-            // to anything deduplicating on `event.id`.
-            this.#socket().message(userId, uid, payload);
-            this.#markDelivered(uid, userId);
-        }
+        if (desktop) this.#sendRewrite(userId, uid, payload);
         const row = await this.stores.notification.getByUid(uid, { userId });
         const app = notification.app as { uid?: unknown } | undefined;
         const appUid =
             row?.app_uid ?? (typeof app?.uid === 'string' ? app.uid : null);
-        this.clients.event.emit(
-            'notif.written',
+        await this.emitWritten(
             {
                 userId,
                 uid,
                 type: registered.type,
                 audience: registered.audience,
                 appUid,
-                value: structuredClone(payload),
+                value: payload,
                 updated: true,
-                sendId: uuidv4(),
+                replacedAt,
+            },
+            { sendId: uuidv4(), rewriteSent: desktop },
+        );
+        return true;
+    }
+
+    /**
+     * Deliver a notification already written: `notif.written` for listeners,
+     * then the live copy. `sendId` names this one send; a delayed send passes
+     * the same one each time it delivers, so a listener can tell a repeat. An
+     * `updated` row is an earlier notification rewritten in place, and
+     * `replacedAt` marks it a replacement. `handlerDepth` is the depth the
+     * write was made at, when no actor in context carries it. `rewriteSent`
+     * says the caller already sent a desktop rewrite. `replayable` marks a send
+     * that may be emitted again under the same `sendId`. Called by extensions.
+     */
+    async emitWritten(
+        written: {
+            userId: number;
+            uid: string;
+            type: string;
+            audience: string;
+            appUid: string | null;
+            value: Record<string, unknown>;
+            updated?: boolean;
+            replacedAt?: number;
+        },
+        opts: {
+            sendId: string;
+            handlerDepth?: number;
+            rewriteSent?: boolean;
+            replayable?: boolean;
+        },
+    ): Promise<void> {
+        const { userId, uid, value: payload, updated } = written;
+        const scope = {
+            type: written.type,
+            audience: written.audience,
+            appUid: written.appUid,
+        };
+        const desktop = inAccountMailbox(scope.type);
+        if (updated && desktop && !opts.rewriteSent)
+            this.#sendRewrite(userId, uid, payload);
+        // Listeners get their own copy; the socket sends `payload`.
+        this.clients.event.emit(
+            'notif.written',
+            {
+                userId,
+                uid,
+                ...scope,
+                value: structuredClone(payload),
+                ...(updated ? { updated: true as const } : {}),
+                sendId: opts.sendId,
+                ...(opts.replayable ? { replayable: true as const } : {}),
             },
             {},
         );
-        // A type kept off the desktop has dispatch as its only live path, so
-        // the rewrite goes out there again, under the same uid.
-        if (!desktop) {
+        // A rewrite of a type kept off the desktop has dispatch as its only
+        // live path, so it goes out there again, under the same uid.
+        if (!updated || !desktop)
             await this.#push(
                 userId,
                 uid,
                 payload,
-                {
-                    type: registered.type,
-                    audience: registered.audience,
-                    appUid,
-                },
-                replacedAt,
+                scope,
+                written.replacedAt,
+                opts.handlerDepth,
             );
-        }
-        return true;
+    }
+
+    /**
+     * Same uid as the original: the client replaces what it is already showing
+     * rather than stacking another copy. It goes straight to the wire rather
+     * than through dispatch — the row was already published, and republishing
+     * it under its own id is a duplicate to anything deduplicating on
+     * `event.id`.
+     */
+    #sendRewrite(
+        userId: number,
+        uid: string,
+        payload: Record<string, unknown>,
+    ): void {
+        this.#socket().message(userId, uid, payload);
+        this.#markDelivered(uid, userId);
     }
 
     /**
@@ -348,17 +405,15 @@ export class NotificationService extends PuterService {
      * Drop notifications past the retention window, in batches, and report how
      * many went. Deleting is all there is to it: nothing is pushed, because a
      * two-week-old row is not news, and a client listing again simply stops
-     * seeing it.
-     *
-     * Every node sweeps. Batches are small and the delete is idempotent, so two
-     * nodes overlapping costs a few empty batches, not correctness.
+     * seeing it. Stops between batches once `signal` aborts.
      */
-    async sweepExpired(): Promise<number> {
+    async sweepExpired(signal?: AbortSignal): Promise<number> {
         const days = this.#retentionDays();
         if (days <= 0) return 0;
 
         let removed = 0;
         for (let pass = 0; pass < RETENTION_MAX_BATCHES; pass++) {
+            if (signal?.aborted) break;
             const batch = await this.stores.notification.deleteCreatedBefore(
                 days,
                 RETENTION_BATCH_SIZE,
@@ -411,6 +466,7 @@ export class NotificationService extends PuterService {
         payload: Record<string, unknown>,
         scope: { type: string; audience: string; appUid: string | null },
         replacedAt?: number,
+        depth?: number,
     ): Promise<void> {
         const desktop = inAccountMailbox(scope.type);
         if (!notificationsFoldInEnabled(this.config)) {
@@ -430,9 +486,8 @@ export class NotificationService extends PuterService {
             return;
         }
 
-        const handlerDepth = handlerDepthOf(
-            Context.get('actor') as Actor | undefined,
-        );
+        const handlerDepth =
+            depth ?? handlerDepthOf(Context.get('actor') as Actor | undefined);
         this.clients.event.emit(
             'notif.created',
             {
@@ -470,17 +525,6 @@ export class NotificationService extends PuterService {
         return Number.isFinite(configured) && configured > 0 ? configured : 0;
     }
 
-    #armRetentionSweep(): void {
-        if (this.#retentionDays() <= 0) return;
-        const sweep = setInterval(() => {
-            void this.sweepExpired().catch((err) => {
-                console.warn('[notification] retention sweep failed', err);
-            });
-        }, RETENTION_SWEEP_INTERVAL_MS);
-        sweep.unref?.();
-        this.#retentionSweep = sweep;
-    }
-
     /**
      * What a reconnecting client missed. One statement marks the batch shown —
      * a per-row update is a round trip per notification on a path that runs

src/backend/services/types.ts

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import type { ScheduleServiceContract } from '../core/schedules.js';
 import type { TeamServiceContract } from '../core/teams.js';
 
 import type { puterClients } from '../clients';
@@ -38,6 +39,8 @@ export interface IPuterServiceInstances {}
 export interface IExtensionServiceInstances {
     /** Teams ships separately; boot guarantees it when `teams_enabled` is on. */
     team?: TeamServiceContract;
+    /** Schedules ship separately; without them their routes answer 503. */
+    schedules?: ScheduleServiceContract;
     [key: string]: unknown;
 }
 

src/backend/services/user/SignupService.test.ts

@@ -172,6 +172,87 @@ describe('SignupService.signup', () => {
     });
 });
 
+describe('SignupService email gate last', () => {
+    /** Signs up with the validate hook asking for `gates`; returns sent emails. */
+    const signupWith = async (gates: Record<string, boolean>) => {
+        const send = vi.fn(async () => true);
+        const clients = server.clients as unknown as { email: unknown };
+        const realEmail = clients.email;
+        clients.email = { send };
+        const require = (_k: string, data: unknown) => {
+            Object.assign(data as Record<string, unknown>, gates);
+        };
+        server.clients.event.on('puter.signup.validate', require);
+        const username = `egl_${slug()}`;
+        try {
+            const user = await server.services.signup.signup({
+                req,
+                body: {
+                    username,
+                    email: `${username}@example.com`,
+                    password: 'correct horse battery',
+                },
+                isTemp: false,
+                fingerprint: null,
+                bonusCode: null,
+            });
+            return { user, send, restore: () => (clients.email = realEmail) };
+        } finally {
+            server.clients.event.off('puter.signup.validate', require);
+        }
+    };
+
+    it('holds the confirmation email until card and phone are cleared, then sends it once', async () => {
+        const { user, send, restore } = await signupWith({
+            requires_phone_verification: true,
+            requires_card_verification: true,
+            email_gate_last: true,
+        });
+        try {
+            expect(send).not.toHaveBeenCalled();
+            const stored = await server.stores.user.getById(user.id, {
+                force: true,
+            });
+            expect(stored!.metadata?.emailGateLast).toBe(true);
+
+            await server.stores.user.update(user.id, {
+                requires_card_verification: 0,
+            });
+            await server.services.signup.sendDeferredConfirmationEmail(user.id);
+            expect(send).not.toHaveBeenCalled();
+
+            await server.stores.user.update(user.id, {
+                requires_phone_verification: 0,
+            });
+            await server.services.signup.sendDeferredConfirmationEmail(user.id);
+            await server.services.signup.sendDeferredConfirmationEmail(user.id);
+            expect(send).toHaveBeenCalledTimes(1);
+            expect(send).toHaveBeenCalledWith(
+                `${user.username}@example.com`,
+                'email_verification_code',
+                { code: stored!.email_confirm_code },
+            );
+        } finally {
+            restore();
+        }
+    });
+
+    it('ignores the flag when there is no phone or card gate', async () => {
+        const { user, send, restore } = await signupWith({
+            email_gate_last: true,
+        });
+        try {
+            expect(send).toHaveBeenCalledTimes(1);
+            const stored = await server.stores.user.getById(user.id, {
+                force: true,
+            });
+            expect(stored!.metadata?.emailGateLast).toBeUndefined();
+        } finally {
+            restore();
+        }
+    });
+});
+
 describe('SignupService.announceSignup', () => {
     it('leaves out the form-only fields for a source that has none', async () => {
         const username = `an_${slug()}`;
@@ -300,3 +381,29 @@ describe('SignupService.confirmEmail', () => {
         expect(after!.email).toBe(other.email);
     });
 });
+
+describe('SignupService.validateEmail', () => {
+    it('refuses Puter mailbox addresses before any listener runs', async () => {
+        const listener = vi.fn();
+        server.clients.event.on('email.validate', listener);
+        try {
+            for (const email of [
+                '[email protected]',
+                '[email protected]',
+                '[email protected]',
+            ]) {
+                await expect(
+                    server.services.signup.validateEmail(email),
+                ).rejects.toMatchObject({ statusCode: 400 });
+            }
+            expect(listener).not.toHaveBeenCalled();
+            await expect(
+                server.services.signup.validateEmail(
+                    `ok-${slug()}@example.com`,
+                ),
+            ).resolves.toBeUndefined();
+        } finally {
+            server.clients.event.off('email.validate', listener);
+        }
+    });
+});

src/backend/services/user/SignupService.ts

@@ -26,12 +26,14 @@ import { HttpError } from '../../core/http/HttpError.js';
 import type { UserRow } from '../../stores/user/UserStore.js';
 import { isOwnedEmailConflict } from '../../stores/user/UserStore.js';
 import { cleanEmail, isBlockedEmail } from '../../util/email.js';
+import { isPuterEmailAddress } from '../email/mailbox.js';
 import {
     bonusCodeInvalidError,
     checkSignupBonus,
     validateSignupBonus,
 } from '../../util/signupBonus.js';
 import { isUsernameTaken } from '../../util/username.js';
+import { isEmailGateLast } from '../../util/verificationOrder.js';
 import {
     generateDefaultFsentries,
     promoteToVerifiedGroup,
@@ -47,6 +49,8 @@ export type SignupGateOutcome =
           requiresEmailConfirmation: boolean;
           requiresPhoneVerification: boolean;
           requiresCardVerification: boolean;
+          /** Card and phone come before email (see `isEmailGateLast`). */
+          emailGateLast: boolean;
       }
     | { verdict: 'bonus_code_invalid' }
     | {
@@ -92,10 +96,20 @@ export const signupClientIp = (req: Request): string | null =>
  */
 export class SignupService extends PuterService {
     /**
-     * Config blocklist (suffix match on the cleaned address), then the
-     * `email.validate` hook. Throws 400 on rejection.
+     * Puter mailbox addresses, the config blocklist (suffix match on the
+     * cleaned address), then the `email.validate` hook. Throws 400 on
+     * rejection.
      */
     async validateEmail(email: string): Promise<void> {
+        // A Puter mailbox can't be an account's own email: it is reached
+        // through an account in the first place.
+        if (isPuterEmailAddress(email.trim())) {
+            throw new HttpError(
+                400,
+                'A Puter email address cannot be used as your account email. Please use a different email address.',
+                { legacyCode: 'email_not_allowed' as never },
+            );
+        }
         if (isBlockedEmail(email, this.config.blockedEmailDomains)) {
             throw new HttpError(400, 'This email is not allowed.', {
                 legacyCode: 'email_not_allowed' as never,
@@ -175,6 +189,9 @@ export class SignupService extends PuterService {
             requires_email_confirmation: false,
             requires_phone_verification: false,
             requires_card_verification: false,
+            // Clear card and phone before email, and send the confirmation
+            // email only then. Meaningless without a phone or card gate.
+            email_gate_last: false,
             message: null as string | null,
             code: null as string | null,
             ...(input.fingerprint !== undefined ? { fingerprint } : {}),
@@ -250,6 +267,9 @@ export class SignupService extends PuterService {
             ),
             requiresPhoneVerification,
             requiresCardVerification,
+            emailGateLast:
+                Boolean(validateEvent.email_gate_last) &&
+                (requiresPhoneVerification || requiresCardVerification),
         };
     }
 
@@ -507,7 +527,10 @@ export class SignupService extends PuterService {
             }
         }
 
-        if (!isTemp && user.requires_email_confirmation) {
+        const emailGateLast = !isTemp && gate.emailGateLast;
+        if (emailGateLast) user = await this.#markEmailGateLast(user);
+
+        if (!isTemp && user.requires_email_confirmation && !emailGateLast) {
             await this.sendConfirmationEmail(
                 user,
                 'signup',
@@ -582,13 +605,46 @@ export class SignupService extends PuterService {
         }
     }
 
+    /** Record that this account clears card and phone before email. */
+    async #markEmailGateLast(user: UserRow): Promise<UserRow> {
+        await this.stores.user.updateMetadata(user.id, { emailGateLast: true });
+        return { ...user, metadata: { ...user.metadata, emailGateLast: true } };
+    }
+
+    /**
+     * Send an email-gate-last account its confirmation email once card and
+     * phone are cleared. Once per account; `/send-confirm-email` resends.
+     */
+    async sendDeferredConfirmationEmail(userId: number): Promise<void> {
+        const user = await this.stores.user.getById(userId, { force: true });
+        if (!user || !isEmailGateLast(user)) return;
+        if (user.metadata?.emailGateLastSent === true) return;
+        if (user.email_confirmed || !user.requires_email_confirmation) return;
+        if (user.requires_phone_verification || user.requires_card_verification)
+            return;
+        await this.stores.user.updateMetadata(user.id, {
+            emailGateLastSent: true,
+        });
+        let code =
+            typeof user.email_confirm_code === 'string'
+                ? user.email_confirm_code
+                : null;
+        if (!code) {
+            code = String(crypto.randomInt(100000, 1000000));
+            await this.stores.user.update(user.id, {
+                email_confirm_code: code,
+            });
+        }
+        await this.sendConfirmationEmail(user, 'deferred', { code });
+    }
+
     /**
      * Send the confirmation code (or link) for the account's address. A lost
      * message is alarmed rather than thrown: the account exists either way.
      */
     async sendConfirmationEmail(
         user: UserRow,
-        stage: 'signup' | 'resend',
+        stage: 'signup' | 'resend' | 'deferred',
         message: { code: string } | { token: string },
     ): Promise<void> {
         if (!this.clients.email || !user.email) return;
@@ -762,7 +818,7 @@ export class SignupService extends PuterService {
      * phone/card gate is outstanding either.
      */
     #confirmationEmailFailed(
-        stage: 'signup' | 'resend',
+        stage: 'signup' | 'resend' | 'deferred',
         user: UserRow,
         cause: unknown,
     ): void {
@@ -774,7 +830,7 @@ export class SignupService extends PuterService {
                   ? 'no transport configured (message dropped)'
                   : String(cause);
         console.warn(
-            `[${stage === 'signup' ? 'signup' : 'send-confirm-email'}] ` +
+            `[${stage === 'resend' ? 'send-confirm-email' : 'signup'}] ` +
                 `confirmation email not delivered: ${detail}`,
         );
         // Best-effort: failing to alarm must not fail the signup.

src/backend/stores/app/AppStore.js

@@ -262,6 +262,19 @@ export class AppStore extends PuterStore {
         return result;
     }
 
+    /**
+     * Cache-free existence check by uid against the primary. Called by
+     * extensions.
+     */
+    async uidExistsOnPrimary(uid) {
+        if (!uid) return false;
+        const rows = await this.clients.db.pread(
+            'SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1',
+            [uid],
+        );
+        return rows.length > 0;
+    }
+
     /** Cache-free existence check against the primary. */
     async existsOnPrimary(appId) {
         if (appId === undefined || appId === null) return false;

src/backend/stores/events/DurableSubscriptionStore.integration.test.ts

@@ -43,6 +43,7 @@ import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENT_SUBSCRIPTION_WIDTHS } from './columnWidths.js';
 import type { DurableSubscriptionInput } from './DurableSubscriptionStore.js';
+import { scheduleToken } from './types.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 
@@ -201,9 +202,9 @@ describe('creating a subscription', () => {
         const { row } = await durable().create(input());
 
         await expect(cache().userHasAny(userId)).resolves.toBe(true);
-        await expect(
-            cache().watchedTokens(userId, [token()]),
-        ).resolves.toEqual([token()]);
+        await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
+            [token()],
+        );
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             row,
         ]);
@@ -468,7 +469,9 @@ describe('the per-account cap', () => {
             input({ expiresAt: Math.floor(Date.now() / 1000) - 60 }),
         );
         await durable().create(input());
-        await expect(durable().countForHolder(userId)).resolves.toMatchObject({ total: 1 });
+        await expect(durable().countForHolder(userId)).resolves.toMatchObject({
+            total: 1,
+        });
     });
 });
 
@@ -547,9 +550,13 @@ describe('rows suspended for good', () => {
         await durable().suspend([failing], 'failures');
         await durable().suspend([revoked], 'permission_revoked');
 
-        const held = await durable().listActiveForHolder(userId, null, {
-            includeSuspended: true,
-        });
+        const { rows: held } = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            0,
+            undefined,
+            { includeSuspended: true },
+        );
         expect(held.map((row) => row.subId).sort()).toEqual(
             [live.subId, failing.subId].sort(),
         );
@@ -565,7 +572,9 @@ describe('rows suspended for good', () => {
             },
         );
 
-        await durable().create(input());
+        const created = await durable().create(input());
+        // The trim's bump, for the caller to publish to the other regions.
+        expect(created.reaped.map((bump) => bump.userId)).toEqual([userId]);
 
         await expect(durable().countForHolder(userId)).resolves.toEqual({
             total: 1,
@@ -600,9 +609,9 @@ describe('removal', () => {
 
         await durable().remove(row);
 
-        await expect(
-            cache().watchedTokens(userId, [token()]),
-        ).resolves.toEqual([token()]);
+        await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
+            [token()],
+        );
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             survivor.row,
         ]);
@@ -744,9 +753,11 @@ describe('the expiry sweep', () => {
         // one back to prove the sweep is what removes it.
         await cache().cacheDurable([expired.row]);
 
-        await expect(durable().sweepExpired(500)).resolves.toBe(1);
+        expect((await durable().sweepExpired(500)).rows).toHaveLength(1);
 
-        await expect(durable().getBySubId(expired.row.subId)).resolves.toBeNull();
+        await expect(
+            durable().getBySubId(expired.row.subId),
+        ).resolves.toBeNull();
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             live.row,
         ]);
@@ -754,19 +765,42 @@ describe('the expiry sweep', () => {
 
     it('leaves a subscription with no expiry alone', async () => {
         await durable().create(input());
-        await expect(durable().sweepExpired(500)).resolves.toBe(0);
+        expect((await durable().sweepExpired(500)).rows).toHaveLength(0);
         await expect(durable().countForHolder(userId)).resolves.toMatchObject({
             total: 1,
         });
     });
 });
 
+describe('a sweep reading a lagging replica', () => {
+    it('keeps a row the primary resumed since, and does not report it', async () => {
+        const { row } = await durable().create(input());
+        await durable().suspend([row], 'permission_revoked');
+        const [stale] = await env.server.clients.db.pread(
+            'SELECT * FROM `event_subscriptions` WHERE `sub_id` = ?',
+            [row.subId],
+        );
+        await durable().resume([row]);
+        // The replica still has it suspended.
+        vi.spyOn(env.server.clients.db, 'read').mockResolvedValueOnce([stale]);
+
+        const reaped = await durable().sweepSuspended(
+            Math.floor(Date.now() / 1000) + 3600,
+            500,
+        );
+
+        expect(reaped.rows).toEqual([]);
+        await expect(durable().getBySubId(row.subId)).resolves.toMatchObject({
+            subId: row.subId,
+            suspendedAt: null,
+        });
+    });
+});
+
 describe('listHolderIdsForApp', () => {
     it('names every distinct holder bound to the app`s handlers', async () => {
         const appUid = `app-${uuidv4()}`;
-        await durable().create(
-            input({ appUid, holderUserId: userId }),
-        );
+        await durable().create(input({ appUid, holderUserId: userId }));
         await durable().create(
             input({
                 appUid,
@@ -774,9 +808,7 @@ describe('listHolderIdsForApp', () => {
             }),
         );
         // A second row for the same holder must not double the name.
-        await durable().create(
-            input({ appUid, holderUserId: userId }),
-        );
+        await durable().create(input({ appUid, holderUserId: userId }));
 
         const holders = await durable().listHolderIdsForApp(appUid);
         expect(holders.sort()).toEqual([userId, otherUserId].sort());
@@ -784,48 +816,44 @@ describe('listHolderIdsForApp', () => {
 
     it('finds a suspended row`s holder too', async () => {
         const appUid = `app-${uuidv4()}`;
-        const { row } = await durable().create(
-            input({ appUid }),
-        );
+        const { row } = await durable().create(input({ appUid }));
         await durable().suspend([row], 'handler_not_found');
 
         expect(await durable().listHolderIdsForApp(appUid)).toEqual([userId]);
     });
 
     it('answers an empty list for an app with nothing bound', async () => {
-        expect(
-            await durable().listHolderIdsForApp(`app-${uuidv4()}`),
-        ).toEqual([]);
+        expect(await durable().listHolderIdsForApp(`app-${uuidv4()}`)).toEqual(
+            [],
+        );
     });
 });
 
 describe('reapForApp', () => {
     it('deletes every row bound to the app and bumps its owner`s generation', async () => {
         const appUid = `app-${uuidv4()}`;
-        const mine = await durable().create(
-            input({ appUid }),
-        );
+        const mine = await durable().create(input({ appUid }));
         const untouched = await durable().create(input());
         const before = await cache().getGeneration(userId);
 
-        await expect(durable().reapForApp(appUid, 500)).resolves.toBe(1);
+        const reaped = await durable().reapForApp(appUid, 500);
+        expect(reaped.rows.map((row) => row.subId)).toEqual([mine.row.subId]);
 
         await expect(durable().getBySubId(mine.row.subId)).resolves.toBeNull();
         await expect(
             durable().getBySubId(untouched.row.subId),
         ).resolves.not.toBeNull();
-        await expect(cache().getGeneration(userId)).resolves.toBeGreaterThan(
-            before,
-        );
+        const after = await cache().getGeneration(userId);
+        expect(after).toBeGreaterThan(before);
+        // The bump the caller publishes to the other regions.
+        expect(reaped.bumps).toEqual([{ userId, generation: after }]);
     });
 
     it('never touches another app`s rows', async () => {
         const appUid = `app-${uuidv4()}`;
         const otherAppUid = `app-${uuidv4()}`;
         await durable().create(input({ appUid }));
-        const theirs = await durable().create(
-            input({ appUid: otherAppUid }),
-        );
+        const theirs = await durable().create(input({ appUid: otherAppUid }));
 
         await durable().reapForApp(appUid, 500);
 
@@ -837,7 +865,7 @@ describe('reapForApp', () => {
     it('answers 0 for an app with nothing to reap', async () => {
         await expect(
             durable().reapForApp(`app-${uuidv4()}`, 500),
-        ).resolves.toBe(0);
+        ).resolves.toEqual({ rows: [], bumps: [] });
     });
 });
 
@@ -904,3 +932,298 @@ describe('suspending', () => {
         expect(second.bumps).toEqual([]);
     });
 });
+
+describe('rows a schedule owns', () => {
+    const scheduleInput = (
+        over: Partial<DurableSubscriptionInput> = {},
+    ): DurableSubscriptionInput => {
+        const uid = uuidv4();
+        return input({
+            subject: scheduleToken(uid),
+            token: scheduleToken(uid),
+            anchorUid: uid,
+            anchorPath: '',
+            appUid: 'app-scheduler',
+            delivery: 'single',
+            handlerName: 'tick',
+            permission: 'events:background',
+            ...over,
+        });
+    };
+
+    it('are created past the durable quota, uncached and with no bump', async () => {
+        const limits = { perUser: 1, perApp: 1 };
+        await durable().create(input({ limits }));
+        const before = await cache().getGeneration(userId);
+
+        const created = await durable().create(scheduleInput({ limits }));
+
+        expect(created.bump).toBeNull();
+        await expect(cache().getGeneration(userId)).resolves.toBe(before);
+        await expect(
+            cache().getForTokens(userId, [created.row.token]),
+        ).resolves.toEqual([]);
+        await expect(
+            durable().getBySubId(created.row.subId),
+        ).resolves.toMatchObject({ token: created.row.token });
+    });
+
+    it('never enter the dispatch cache or what a cold region rebuilds', async () => {
+        const { row } = await durable().create(scheduleInput());
+        await expect(
+            durable().listDeliverableForOwner(userId),
+        ).resolves.toEqual([]);
+
+        await cache().markRegionCold(userId);
+        await durable().warmRegion(userId);
+        await expect(cache().userHasAny(userId)).resolves.toBe(false);
+
+        await durable().suspend([row], 'no_credit');
+        await durable().resume([
+            { ...row, suspendedAt: 1, suspendedReason: 'no_credit' },
+        ]);
+        await expect(
+            cache().getForTokens(userId, [row.token]),
+        ).resolves.toEqual([]);
+        await expect(cache().userHasAny(userId)).resolves.toBe(false);
+    });
+
+    it('bump no generation when suspended or resumed', async () => {
+        const { row } = await durable().create(scheduleInput());
+        const before = await cache().getGeneration(userId);
+
+        const { suspended, bumps } = await durable().suspend([row], 'failures');
+        expect(suspended).toHaveLength(1);
+        expect(bumps).toEqual([]);
+        await expect(durable().resume(suspended)).resolves.toEqual([]);
+        await expect(cache().getGeneration(userId)).resolves.toBe(before);
+    });
+
+    it('count toward no durable quota, and are not in the durable listing', async () => {
+        await durable().create(input());
+        await durable().create(scheduleInput());
+
+        await expect(durable().countForHolder(userId)).resolves.toEqual({
+            total: 1,
+            forApp: 0,
+            terminal: 0,
+        });
+        const listed = await durable().listForHolder(userId, {
+            includeTotal: true,
+        });
+        expect(listed.total).toBe(1);
+        expect(listed.items.map((row) => row.token)).toEqual([token()]);
+    });
+
+    it('are kept when the oldest revoked rows are trimmed', async () => {
+        const base = Math.floor(Date.now() / 1000) - 10_000;
+        const { row: scheduled } = await durable().create(scheduleInput());
+        await env.server.clients.db.write(
+            'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
+                '`suspended_reason` = ? WHERE `sub_id` = ?',
+            [base - 1, 'permission_revoked', scheduled.subId],
+        );
+        for (let i = 0; i < EVENTS_REVOKED_SUBSCRIPTIONS_KEPT + 1; i++) {
+            const { row } = await durable().create(input());
+            await env.server.clients.db.write(
+                'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
+                    '`suspended_reason` = ? WHERE `sub_id` = ?',
+                [base + i, 'permission_revoked', row.subId],
+            );
+        }
+
+        await durable().create(input());
+
+        await expect(
+            durable().getBySubId(scheduled.subId),
+        ).resolves.not.toBeNull();
+        await expect(durable().countForHolder(userId)).resolves.toMatchObject({
+            terminal: EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
+        });
+    });
+
+    it('are read, a page at a time, by what settles a holder', async () => {
+        const made: string[] = [];
+        for (let i = 0; i < 3; i++)
+            made.push((await durable().create(scheduleInput())).row.subId);
+        made.push((await durable().create(input())).row.subId);
+
+        const first = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            0,
+            2,
+        );
+        expect(first.rows.map((row) => row.subId)).toEqual(made.slice(0, 2));
+        expect(first.nextId).not.toBeNull();
+        const second = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            first.nextId!,
+            2,
+        );
+        expect(second.rows.map((row) => row.subId)).toEqual(made.slice(2));
+        // A full page always says to look again; the next one is empty.
+        const third = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            second.nextId ?? Number.MAX_SAFE_INTEGER,
+            2,
+        );
+        expect(third).toEqual({ rows: [], nextId: null });
+    });
+
+    it('are read, a page at a time, by what a top-up resumes', async () => {
+        const rows = [];
+        for (let i = 0; i < 3; i++)
+            rows.push((await durable().create(scheduleInput())).row);
+        await durable().suspend(rows, 'no_credit');
+
+        const first = await durable().listSuspendedForHolderPage(
+            userId,
+            'no_credit',
+            0,
+            2,
+        );
+        expect(first.rows).toHaveLength(2);
+        const second = await durable().listSuspendedForHolderPage(
+            userId,
+            'no_credit',
+            first.nextId!,
+            2,
+        );
+        expect(second.rows.map((row) => row.subId)).toEqual([rows[2].subId]);
+        expect(second.nextId).toBeNull();
+    });
+
+    it('are found by handler and by app, like any other row', async () => {
+        const { row } = await durable().create(scheduleInput());
+        await expect(
+            durable().listByHandler('app-scheduler', 'tick'),
+        ).resolves.toEqual([expect.objectContaining({ subId: row.subId })]);
+        await expect(
+            durable().listHolderIdsForApp('app-scheduler'),
+        ).resolves.toEqual([userId]);
+    });
+
+    it('come back from a reap without touching a backlog', async () => {
+        const { row } = await durable().create(scheduleInput());
+        const regular = await durable().create(
+            input({ appUid: 'app-scheduler' }),
+        );
+        const purge = vi.spyOn(env.server.stores.pendingDelivery, 'purge');
+        try {
+            const reaped = await durable().reapForApp('app-scheduler', 500);
+            expect(reaped.rows.map((r) => r.subId).sort()).toEqual(
+                [row.subId, regular.row.subId].sort(),
+            );
+            expect(purge.mock.calls.map(([subId]) => subId)).toEqual([
+                regular.row.subId,
+            ]);
+            // Only the cached row's owner was bumped.
+            expect(reaped.bumps).toHaveLength(1);
+        } finally {
+            purge.mockRestore();
+        }
+    });
+
+    it('are deleted by their schedule`s id, whether or not it recorded the row', async () => {
+        const uid = uuidv4();
+        await durable().create(
+            scheduleInput({
+                subject: scheduleToken(uid),
+                token: scheduleToken(uid),
+                anchorUid: uid,
+            }),
+        );
+        await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
+            true,
+        );
+        await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
+            false,
+        );
+    });
+});
+
+describe('statements a schedule store composes', () => {
+    const createScheduleRow = async () => {
+        const uid = uuidv4();
+        const { row } = await durable().create(
+            input({
+                subject: scheduleToken(uid),
+                token: scheduleToken(uid),
+                anchorUid: uid,
+                anchorPath: '',
+                appUid: 'app-scheduler',
+                delivery: 'single',
+                handlerName: 'tick',
+                context: '{"region":"eu"}',
+                permission: 'events:background',
+            }),
+        );
+        return { uid, row };
+    };
+
+    it('delete a schedule`s row only while the guard holds', async () => {
+        const { uid, row } = await createScheduleRow();
+        const db = env.server.clients.db;
+
+        await db.batchWrite([
+            durable().scheduleSubscriptionDeleteEntry(uid, {
+                table: 'user',
+                where: '`id` = ?',
+                values: [-1],
+            }),
+        ]);
+        await expect(durable().getBySubId(row.subId)).resolves.not.toBeNull();
+
+        await db.batchWrite([
+            durable().scheduleSubscriptionDeleteEntry(uid, {
+                table: 'user',
+                where: '`id` = ?',
+                values: [userId],
+            }),
+        ]);
+        await expect(durable().getBySubId(row.subId)).resolves.toBeNull();
+    });
+
+    it('join a schedule`s row under the sub_* columns', async () => {
+        const { row } = await createScheduleRow();
+        const join = durable().scheduleSubscriptionJoin('sub', 'x.`sub_id`');
+        const read = (subId: string) =>
+            env.server.clients.db.pread(
+                `SELECT ${join.columns} FROM (SELECT ? AS \`sub_id\`) x ${join.join}`,
+                [subId],
+            );
+
+        const [found] = await read(row.subId);
+        expect(found).toMatchObject({
+            sub_suspended_at: null,
+            sub_suspended_reason: null,
+            sub_handler_name: 'tick',
+        });
+        expect(Number(found.sub_row_id)).toBeGreaterThan(0);
+        expect(found.sub_context).toBe('{"region":"eu"}');
+
+        const [missing] = await read('no-such-sub');
+        expect(missing).toMatchObject({
+            sub_row_id: null,
+            sub_handler_name: null,
+        });
+    });
+
+    it('tell a schedule whose row is gone', async () => {
+        const { uid, row } = await createScheduleRow();
+        const absent = async (subId: string) => {
+            const [result] = await env.server.clients.db.pread(
+                `SELECT CASE WHEN ${durable().scheduleSubscriptionAbsent('?')} THEN 1 ELSE 0 END AS \`absent\``,
+                [subId],
+            );
+            return Number(result.absent);
+        };
+
+        await expect(absent(row.subId)).resolves.toBe(0);
+        await durable().removeScheduleSubscription(uid);
+        await expect(absent(row.subId)).resolves.toBe(1);
+    });
+});

src/backend/stores/events/DurableSubscriptionStore.ts

@@ -23,7 +23,9 @@ import {
     EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
     type TieredQuota,
 } from '../../controllers/events/limits.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { HttpError } from '../../core/http/HttpError.js';
+import type { SqlGuard } from '../../core/schedules.js';
 import type { DeliveryClass } from '../../services/events/registry.js';
 import type { FsOp } from '../../services/events/subjects.js';
 import {
@@ -37,7 +39,13 @@ import {
     EVENT_SUBSCRIPTION_WIDTHS,
 } from './columnWidths.js';
 import type { GenerationBump } from './EventSubscriptionStore.js';
-import type { DurableSubscription, SubscriptionPermission } from './types.js';
+import {
+    isScheduleRow,
+    SCHEDULE_TOKEN_PREFIX,
+    scheduleToken,
+    type DurableSubscription,
+    type SubscriptionPermission,
+} from './types.js';
 
 /**
  * Subscriptions that outlive the connection that made them.
@@ -66,6 +74,9 @@ export const DURABLE_CONTEXT_MAX_BYTES = 4096;
 export const DURABLE_LIST_DEFAULT_LIMIT = 50;
 export const DURABLE_LIST_LIMIT_CAP = 200;
 
+/** Rows one page of a holder-wide settle takes. */
+export const HOLDER_SETTLE_BATCH = 500;
+
 /**
  * Rows one handler-lifecycle pass takes. A removal suspends its dependents in
  * batches so a widely-used name cannot make one call hold the whole set.
@@ -101,6 +112,15 @@ export interface DurableSubscriptionInput {
     limits?: TieredQuota;
 }
 
+/**
+ * What a reap deleted, and the generation it bumped per owner. The caller
+ * publishes the bumps, since the reap only invalidates this region.
+ */
+export interface DurableReap {
+    rows: DurableSubscription[];
+    bumps: GenerationBump[];
+}
+
 /** One keyset page of rows, and where the next page starts. */
 export interface DurablePage {
     rows: DurableSubscription[];
@@ -148,6 +168,13 @@ const TERMINAL_CLAUSE =
     '(`suspended_at` IS NOT NULL AND `suspended_reason` IN (' +
     `${TERMINAL_SUSPENDED_REASONS.map(() => '?').join(', ')}))`;
 
+/**
+ * Leaves out the rows schedules own: they have their own caps and listing, and
+ * never enter the dispatch cache. Binds {@link SCHEDULE_TOKEN_PATTERN}.
+ */
+const NOT_SCHEDULE_CLAUSE = '`token` NOT LIKE ?';
+const SCHEDULE_TOKEN_PATTERN = `${SCHEDULE_TOKEN_PREFIX}%`;
+
 /** Where a row is moving to when its anchor is deleted under it. */
 export interface ReanchorInput {
     token: string;
@@ -167,6 +194,12 @@ const contextTooLarge = (): HttpError =>
         { legacyCode: 'events_context_too_large' },
     );
 
+export const assertDurableContext = (context: string | null): void => {
+    if (context === null) return;
+    if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
+        throw contextTooLarge();
+};
+
 const quotaReached = (limit: number, scope: 'account' | 'app'): HttpError =>
     new HttpError(
         429,
@@ -207,6 +240,16 @@ const quotaRefusal = (
     return null;
 };
 
+/**
+ * A created row; `reaped` holds the bumps of terminal rows it trimmed. A
+ * schedule's row is never cached, so it comes back with no bump.
+ */
+export interface DurableCreated {
+    row: DurableSubscription;
+    bump: GenerationBump | null;
+    reaped: GenerationBump[];
+}
+
 /**
  * The quota refusal for a create that concurrent creates beat to the last slot
  * after its own insert. Its row is already gone again; `bump` still has to be
@@ -291,11 +334,13 @@ export class DurableSubscriptionStore extends PuterStore {
      * Insert one subscription and put it in this region's cache before
      * returning. The generation bump is what tells every other region, and the
      * caller broadcasts it.
+     *
+     * A schedule's row skips the durable quotas and the cache, so it comes back
+     * with no bump: its caps are the schedule store's.
      */
-    async create(
-        input: DurableSubscriptionInput,
-    ): Promise<{ row: DurableSubscription; bump: GenerationBump }> {
-        this.#assertContext(input.context);
+    async create(input: DurableSubscriptionInput): Promise<DurableCreated> {
+        assertDurableContext(input.context);
+        const forSchedule = isScheduleRow(input);
         assertColumnWidths(EVENT_SUBSCRIPTION_WIDTHS, {
             token: input.token,
             appUid: input.appUid,
@@ -307,12 +352,14 @@ export class DurableSubscriptionStore extends PuterStore {
             permission: input.permission,
         });
 
-        const refusal = quotaRefusal(
-            input,
-            await this.countForHolder(input.holderUserId, input.appUid),
-            0,
-        );
-        if (refusal) throw refusal;
+        if (!forSchedule) {
+            const refusal = quotaRefusal(
+                input,
+                await this.countForHolder(input.holderUserId, input.appUid),
+                0,
+            );
+            if (refusal) throw refusal;
+        }
 
         const row: DurableSubscription = {
             durable: true,
@@ -358,6 +405,7 @@ export class DurableSubscriptionStore extends PuterStore {
             expires_at: row.expiresAt,
             created_at: row.createdAt,
         });
+        if (forSchedule) return { row, bump: null, reaped: [] };
 
         // Concurrent creates can all pass the count above. Counted again with
         // this row in it, each one past the cap takes itself back out — at
@@ -368,18 +416,20 @@ export class DurableSubscriptionStore extends PuterStore {
         );
         const lost = quotaRefusal(input, held, 1);
         if (lost) throw new DurableQuotaRaceLost(lost, await this.remove(row));
-        if (held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT)
-            await this.#dropOldestTerminal(
-                input.holderUserId,
-                held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
-            );
+        const reaped =
+            held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT
+                ? await this.#dropOldestTerminal(
+                      input.holderUserId,
+                      held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
+                  )
+                : [];
 
         // Write-through, and over the whole owner rather than the one row: it
         // costs the same indexed read as a warm would, on a path rate-limited
         // to a few calls a minute, and it leaves this region needing nothing
         // from the table before it can deliver.
         await this.#rebuildRegion(row.ownerUserId);
-        return { row, bump: await this.#bump(row.ownerUserId) };
+        return { row, bump: await this.#bump(row.ownerUserId), reaped };
     }
 
     /**
@@ -439,8 +489,9 @@ export class DurableSubscriptionStore extends PuterStore {
                 });
         }
 
-        await this.stores.eventSubscription.dropDurables(suspended);
-        const owners = new Set(suspended.map((row) => row.ownerUserId));
+        const cached = suspended.filter((row) => !isScheduleRow(row));
+        await this.stores.eventSubscription.dropDurables(cached);
+        const owners = new Set(cached.map((row) => row.ownerUserId));
         const bumps = await Promise.all(
             [...owners].map((owner) => this.#bump(owner)),
         );
@@ -468,6 +519,7 @@ export class DurableSubscriptionStore extends PuterStore {
         // owner from the first row it is given.
         const byOwner = new Map<number, DurableSubscription[]>();
         for (const row of rows) {
+            if (isScheduleRow(row)) continue;
             const live: DurableSubscription = {
                 ...row,
                 suspendedAt: null,
@@ -541,7 +593,7 @@ export class DurableSubscriptionStore extends PuterStore {
         row: DurableSubscription,
         context: string | null,
     ): Promise<{ row: DurableSubscription; bump: GenerationBump } | null> {
-        this.#assertContext(context);
+        assertDurableContext(context);
         const written = await this.clients.db.write(
             `UPDATE \`${TABLE}\` SET \`context\` = ? WHERE \`sub_id\` = ?`,
             [context, row.subId],
@@ -573,8 +625,12 @@ export class DurableSubscriptionStore extends PuterStore {
      * Reap rows past their expiry, dropping each from the cache so the region
      * stops delivering against it without waiting for a rebuild.
      */
-    async sweepExpired(batchSize: number): Promise<number> {
-        return this.#reap(await this.#listExpired(nowSeconds(), batchSize));
+    async sweepExpired(batchSize: number): Promise<DurableReap> {
+        const cutoff = nowSeconds();
+        return this.#reap(await this.#listExpired(cutoff, batchSize), {
+            where: '`expires_at` IS NOT NULL AND `expires_at` <= ?',
+            values: [cutoff],
+        });
     }
 
     /**
@@ -582,8 +638,14 @@ export class DurableSubscriptionStore extends PuterStore {
      * never resumes is a row kept only so its holder can see why it stopped,
      * and that answer has a shelf life.
      */
-    async sweepSuspended(cutoff: number, batchSize: number): Promise<number> {
-        return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize));
+    async sweepSuspended(
+        cutoff: number,
+        batchSize: number,
+    ): Promise<DurableReap> {
+        return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize), {
+            where: '`suspended_at` IS NOT NULL AND `suspended_at` <= ?',
+            values: [cutoff],
+        });
     }
 
     // -- Reads -------------------------------------------------------
@@ -622,8 +684,16 @@ export class DurableSubscriptionStore extends PuterStore {
 
         // The scope half of the predicate is what the total counts over; the
         // cursor half only positions one page inside it.
-        const scope = ['`holder_user_id` = ?', this.#unexpiredClause()];
-        const scopeParams: unknown[] = [holderUserId, nowSeconds()];
+        const scope = [
+            '`holder_user_id` = ?',
+            this.#unexpiredClause(),
+            NOT_SCHEDULE_CLAUSE,
+        ];
+        const scopeParams: unknown[] = [
+            holderUserId,
+            nowSeconds(),
+            SCHEDULE_TOKEN_PATTERN,
+        ];
         if (options.appUid !== undefined && options.appUid !== null) {
             scope.push('`app_uid` = ?');
             scopeParams.push(options.appUid);
@@ -675,7 +745,8 @@ export class DurableSubscriptionStore extends PuterStore {
                 `SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 0 ` +
                 'WHEN `app_uid` = ? THEN 1 ELSE 0 END) AS `for_app`, ' +
                 `SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 1 ELSE 0 END) AS \`terminal\` ` +
-                `FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()}`,
+                `FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()} ` +
+                `AND ${NOT_SCHEDULE_CLAUSE}`,
             [
                 ...terminal,
                 ...terminal,
@@ -683,6 +754,7 @@ export class DurableSubscriptionStore extends PuterStore {
                 ...terminal,
                 holderUserId,
                 nowSeconds(),
+                SCHEDULE_TOKEN_PATTERN,
             ],
         );
         return {
@@ -693,22 +765,26 @@ export class DurableSubscriptionStore extends PuterStore {
     }
 
     /**
-     * The holder's live rows, which is what a revocation has to consider. Same
-     * index the listing and the quota use — passing `appUid` narrows to one
-     * app's rows, which is what a grant made to that app can have authorized.
+     * The holder's live rows, which is what a revocation has to consider, one
+     * keyset page at a time. Same index the listing and the quota use — passing
+     * `appUid` narrows to one app's rows, which is what a grant made to that
+     * app can have authorized. Schedule rows are included: a holder can hold
+     * far more of them than the durable quota allows, which is why this pages.
      *
      * `includeSuspended` also returns rows out of service for a reason that
      * lifts. A revocation needs them: those reasons hold a backlog, and a row
      * skipped here comes back in service on the next resume and hands it over.
+     * Rows suspended for good are left out.
      *
-     * Bounded by the per-account quota, so the whole set fits one read. Rows
-     * suspended for good hold no slot, so they are left out.
+     * `excludeSchedules` is for a settle no schedule row can match.
      */
-    async listActiveForHolder(
+    async listActiveForHolderPage(
         holderUserId: number,
         appUid: string | null,
-        opts: { includeSuspended?: boolean } = {},
-    ): Promise<DurableSubscription[]> {
+        afterId: number,
+        batchSize: number = HOLDER_SETTLE_BATCH,
+        opts: { includeSuspended?: boolean; excludeSchedules?: boolean } = {},
+    ): Promise<DurablePage> {
         const where = [
             '`holder_user_id` = ?',
             opts.includeSuspended
@@ -721,16 +797,17 @@ export class DurableSubscriptionStore extends PuterStore {
             ...(opts.includeSuspended ? TERMINAL_SUSPENDED_REASONS : []),
             nowSeconds(),
         ];
+        if (opts.excludeSchedules) {
+            where.push(NOT_SCHEDULE_CLAUSE);
+            params.push(SCHEDULE_TOKEN_PATTERN);
+        }
         if (appUid !== null) {
             where.push('`app_uid` = ?');
             params.push(appUid);
         }
-        const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                `WHERE ${where.join(' AND ')} ORDER BY \`id\` LIMIT ?`,
-            [...params, EVENTS_DURABLE_SUBSCRIPTIONS_MAX],
-        );
-        return rows.map(toRow);
+        return this.#page(where, params, afterId, batchSize, {
+            primary: true,
+        });
     }
 
     /**
@@ -768,25 +845,27 @@ export class DurableSubscriptionStore extends PuterStore {
 
     /**
      * One holder's rows suspended for a given reason — what a resume condition
-     * that belongs to the account rather than to a handler releases.
+     * that belongs to the account rather than to a handler releases — one
+     * keyset page at a time.
      */
-    async listSuspendedForHolder(
+    async listSuspendedForHolderPage(
         holderUserId: number,
         reason: SuspendedReason,
-    ): Promise<DurableSubscription[]> {
-        const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                'WHERE `holder_user_id` = ? AND `suspended_at` IS NOT NULL ' +
-                `AND \`suspended_reason\` = ? AND ${this.#unexpiredClause()} ` +
-                'ORDER BY `id` LIMIT ?',
+        afterId: number,
+        batchSize: number = HOLDER_SETTLE_BATCH,
+    ): Promise<DurablePage> {
+        return this.#page(
             [
-                holderUserId,
-                reason,
-                nowSeconds(),
-                EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
+                '`holder_user_id` = ?',
+                '`suspended_at` IS NOT NULL',
+                '`suspended_reason` = ?',
+                this.#unexpiredClause(),
             ],
+            [holderUserId, reason, nowSeconds()],
+            afterId,
+            batchSize,
+            { primary: true },
         );
-        return rows.map(toRow);
     }
 
     /**
@@ -819,8 +898,8 @@ export class DurableSubscriptionStore extends PuterStore {
         const rows = await this.clients.db.pread(
             `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                 'WHERE `owner_user_id` = ? AND `suspended_at` IS NULL AND ' +
-                `${this.#unexpiredClause()}`,
-            [ownerUserId, nowSeconds()],
+                `${this.#unexpiredClause()} AND ${NOT_SCHEDULE_CLAUSE}`,
+            [ownerUserId, nowSeconds(), SCHEDULE_TOKEN_PATTERN],
         );
         return rows.map(toRow);
     }
@@ -846,30 +925,96 @@ export class DurableSubscriptionStore extends PuterStore {
      * What app deletion reaps: `#reap` already purges the backlog, drops the
      * cache and bumps a generation per owner, so this is the whole of it.
      */
-    async reapForApp(appUid: string, batchSize: number): Promise<number> {
+    async reapForApp(appUid: string, batchSize: number): Promise<DurableReap> {
         return this.#reap(await this.#listForApp(appUid, batchSize));
     }
 
+    /**
+     * Delete the row a schedule owns, whether or not the schedule ever recorded
+     * its id: a create that crashed between the two inserts leaves one it
+     * didn't. Never cached, so there is nothing to drop or bump. Called by
+     * extensions.
+     */
+    async removeScheduleSubscription(scheduleUid: string): Promise<boolean> {
+        const written = await this.clients.db.write(
+            `DELETE FROM \`${TABLE}\` WHERE \`token\` = ?`,
+            [scheduleToken(scheduleUid)],
+        );
+        return written.anyRowsAffected;
+    }
+
+    // Statements and fragments for the schedules' own transactions and reads,
+    // so no SQL outside this store names the table. Called by extensions.
+
+    /**
+     * Delete the row a schedule owns, only while `guard` holds: for a batch
+     * that ends the schedule in the same transaction.
+     */
+    scheduleSubscriptionDeleteEntry(
+        scheduleUid: string,
+        guard: SqlGuard,
+    ): BatchEntry {
+        return {
+            statement:
+                `DELETE FROM \`${TABLE}\` WHERE \`token\` = ? ` +
+                `AND EXISTS (SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where})`,
+            values: [scheduleToken(scheduleUid), ...guard.values],
+        };
+    }
+
+    /**
+     * A `LEFT JOIN` of the row a schedule owns, as `alias`, on `subIdColumn`,
+     * and the columns it selects: `sub_row_id`, `sub_suspended_at`,
+     * `sub_suspended_reason`, `sub_handler_name` and `sub_context`.
+     */
+    scheduleSubscriptionJoin(
+        alias: string,
+        subIdColumn: string,
+    ): { columns: string; join: string } {
+        return {
+            columns:
+                `${alias}.\`id\` AS \`sub_row_id\`, ` +
+                `${alias}.\`suspended_at\` AS \`sub_suspended_at\`, ` +
+                `${alias}.\`suspended_reason\` AS \`sub_suspended_reason\`, ` +
+                `${alias}.\`handler_name\` AS \`sub_handler_name\`, ` +
+                `${alias}.\`context\` AS \`sub_context\``,
+            join: `LEFT JOIN \`${TABLE}\` ${alias} ON ${alias}.\`sub_id\` = ${subIdColumn}`,
+        };
+    }
+
+    /** A condition true when no subscription row has the id `subIdExpr`. */
+    scheduleSubscriptionAbsent(subIdExpr: string): string {
+        return `NOT EXISTS (SELECT 1 FROM \`${TABLE}\` e WHERE e.\`sub_id\` = ${subIdExpr})`;
+    }
+
     // -- Internals ---------------------------------------------------
 
     /**
-     * Delete a holder's oldest rows suspended for good, past what `list` keeps.
-     * Best-effort: the next subscribe tries again.
+     * Delete a holder's oldest rows suspended for good, past what `list` keeps,
+     * and return the reap's bumps. Best-effort: the next subscribe tries
+     * again.
      */
     async #dropOldestTerminal(
         holderUserId: number,
         count: number,
-    ): Promise<void> {
+    ): Promise<GenerationBump[]> {
         try {
             const rows = await this.clients.db.pread(
                 `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                     `WHERE \`holder_user_id\` = ? AND ${TERMINAL_CLAUSE} ` +
+                    `AND ${NOT_SCHEDULE_CLAUSE} ` +
                     'ORDER BY `suspended_at`, `id` LIMIT ?',
-                [holderUserId, ...TERMINAL_SUSPENDED_REASONS, count],
+                [
+                    holderUserId,
+                    ...TERMINAL_SUSPENDED_REASONS,
+                    SCHEDULE_TOKEN_PATTERN,
+                    count,
+                ],
             );
-            await this.#reap(rows.map(toRow));
+            return (await this.#reap(rows.map(toRow))).bumps;
         } catch (err) {
             console.warn('[events] could not drop old revoked rows', err);
+            return [];
         }
     }
 
@@ -891,15 +1036,23 @@ export class DurableSubscriptionStore extends PuterStore {
         return rows.map(toRow);
     }
 
-    /** One page of a whole-table scan, ordered and positioned by primary key. */
+    /**
+     * One keyset page, ordered and positioned by primary key. The replica
+     * unless `primary`, which settles need: a lagging read would skip a row
+     * just created.
+     */
     async #page(
         where: string[],
         params: unknown[],
         afterId: number,
         batchSize: number,
+        opts: { primary?: boolean } = {},
     ): Promise<DurablePage> {
         const limit = Math.max(1, Math.floor(batchSize));
-        const rows = await this.clients.db.read(
+        const read = opts.primary
+            ? this.clients.db.pread.bind(this.clients.db)
+            : this.clients.db.read.bind(this.clients.db);
+        const rows = await read(
             `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                 `WHERE ${[...where, '`id` > ?'].join(' AND ')} ` +
                 'ORDER BY `id` LIMIT ?',
@@ -952,30 +1105,56 @@ export class DurableSubscriptionStore extends PuterStore {
         return rows.map(toRow);
     }
 
-    /** Delete a batch and leave no region delivering against any of it. */
-    async #reap(rows: readonly DurableSubscription[]): Promise<number> {
-        if (rows.length === 0) return 0;
+    /**
+     * Delete a batch and stop this region delivering against any of it. Other
+     * regions stop once the caller publishes the returned bumps; the deleted
+     * rows come back so the caller can end the schedules among them.
+     *
+     * `stillDue` re-checks candidates read from a replica on the primary, so a
+     * row resumed or extended since is kept and not reported.
+     */
+    async #reap(
+        candidates: readonly DurableSubscription[],
+        stillDue?: { where: string; values: unknown[] },
+    ): Promise<DurableReap> {
+        if (candidates.length === 0) return { rows: [], bumps: [] };
 
+        const ids = candidates.map((row) => row.subId);
+        const inList = `(${ids.map(() => '?').join(', ')})`;
         await this.clients.db.write(
-            `DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ` +
-                `(${rows.map(() => '?').join(', ')})`,
-            rows.map((row) => row.subId),
+            `DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}` +
+                (stillDue ? ` AND ${stillDue.where}` : ''),
+            [...ids, ...(stillDue?.values ?? [])],
         );
-
-        await this.stores.eventSubscription.dropDurables(rows);
-        for (const row of rows) {
-            // Whatever was still owed to a row that no longer exists.
-            await this.stores.pendingDelivery.purge(row.subId).catch(() => {});
+        let rows = candidates;
+        if (stillDue) {
+            const kept = new Set(
+                (
+                    await this.clients.db.pread(
+                        `SELECT \`sub_id\` FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}`,
+                        ids,
+                    )
+                ).map((row) => String(row.sub_id)),
+            );
+            rows = candidates.filter((row) => !kept.has(row.subId));
+            if (rows.length === 0) return { rows: [], bumps: [] };
         }
-        const owners = new Set(rows.map((row) => row.ownerUserId));
-        for (const ownerUserId of owners) await this.#bump(ownerUserId);
-        return rows.length;
-    }
 
-    #assertContext(context: string | null): void {
-        if (context === null) return;
-        if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
-            throw contextTooLarge();
+        const cached = rows.filter((row) => !isScheduleRow(row));
+        await this.stores.eventSubscription.dropDurables(cached);
+        for (const row of cached) {
+            // Whatever was still owed to a row that no longer exists.
+            try {
+                await this.stores.pendingDelivery.purge(row.subId);
+            } catch {
+                // The pending sweep purges it once it finds the row gone.
+            }
+        }
+        const owners = new Set(cached.map((row) => row.ownerUserId));
+        const bumps: GenerationBump[] = [];
+        for (const ownerUserId of owners)
+            bumps.push(await this.#bump(ownerUserId));
+        return { rows: [...rows], bumps };
     }
 
     async #bump(ownerUserId: number): Promise<GenerationBump> {

src/backend/stores/events/EventHandlerStore.integration.test.ts

@@ -55,7 +55,10 @@ const OTHER_SOURCE = 'async ({ event, ctx }) => { console.log(ctx.url); }';
 const codeOf = (code: string) => (err: unknown) =>
     isHttpError(err) && err.legacyCode === code;
 
-/** A minimal `apps` row, owned by `ownerUserId`, for the events-workers reads to join against. */
+/**
+ * A minimal `apps` row, owned by `ownerUserId`, for the events-workers reads to
+ * join against.
+ */
 const makeApp = async (owner: number): Promise<string> => {
     const uid = `app-${uuidv4()}`;
     await env.server.clients.db.write(
@@ -122,7 +125,11 @@ describe('publishing a handler', () => {
     });
 
     it('refuses different source from a publisher that did not name the base', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
 
         await expect(
             handlers().publish({
@@ -179,7 +186,11 @@ describe('publishing a handler', () => {
     });
 
     it('takes the name outright for a publisher that asked to replace', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
 
         const replaced = await handlers().publish({
             appUid,
@@ -236,9 +247,10 @@ describe('publishing a handler', () => {
         // The row reflects whichever one actually won — never a mix of the
         // two, and never silently both.
         const stored = await handlers().getByName(appUid, 'ingestUpload');
-        expect([OTHER_SOURCE, 'async () => { /* the other racer */ }']).toContain(
-            stored!.source,
-        );
+        expect([
+            OTHER_SOURCE,
+            'async () => { /* the other racer */ }',
+        ]).toContain(stored!.source);
     });
 
     it('never lets a create-vs-create race leak a raw database error', async () => {
@@ -271,7 +283,11 @@ describe('publishing a handler', () => {
     });
 
     it('scopes the name to one app', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
         const other = await handlers().publish({
             appUid: otherAppUid,
             name: 'ingestUpload',
@@ -285,7 +301,13 @@ describe('publishing a handler', () => {
     });
 
     it('refuses a name that is not an addressable identifier', async () => {
-        for (const name of ['', ' leading', 'has space', '-dash', 'x'.repeat(129)]) {
+        for (const name of [
+            '',
+            ' leading',
+            'has space',
+            '-dash',
+            'x'.repeat(129),
+        ]) {
             await expect(
                 handlers().publish({ appUid, name, source: SOURCE }),
             ).rejects.toSatisfy(codeOf('events_handler_name_invalid'));
@@ -309,7 +331,11 @@ describe('publishing a handler', () => {
 
 describe('listing handlers', () => {
     it('reports names, hashes and dependents, never source', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
         await handlers().publish({
             appUid,
             name: 'indexDocument',
@@ -327,13 +353,15 @@ describe('listing handlers', () => {
             hash: hashContent(SOURCE),
             subscriptions: 0,
         });
-        expect(
-            listed.every((row) => !('source' in row)),
-        ).toBe(true);
+        expect(listed.every((row) => !('source' in row))).toBe(true);
     });
 
     it('counts the subscriptions each name is carrying', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
         for (const suffix of ['a', 'b']) {
             await env.server.clients.db.write(
                 'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, ' +
@@ -365,7 +393,11 @@ describe('listing handlers', () => {
 
 describe('removing a handler', () => {
     it('drops the row and reports what it was', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
 
         await expect(
             handlers().remove(appUid, 'ingestUpload'),
@@ -437,6 +469,136 @@ describe('the set a delivery addresses', () => {
         }
     });
 
+    it('reads a miss from the replica when asked to, and caches it', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        const pread = vi.spyOn(env.server.clients.db, 'pread');
+        const read = vi.spyOn(env.server.clients.db, 'read');
+        try {
+            const first = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(first.map((h) => h.name)).toEqual(['a']);
+            await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(pread).not.toHaveBeenCalled();
+            expect(read).toHaveBeenCalledTimes(1);
+        } finally {
+            pread.mockRestore();
+            read.mockRestore();
+        }
+    });
+
+    it('never hands the delivery path a set a lagging replica read cached', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        const fresh = 'async ({ event }) => event.path';
+        // The replica still has the old source when a schedule run reads it,
+        // right after a publish dropped the caches.
+        const stale = await handlers().setForApp(cachedApp);
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: fresh,
+            replace: true,
+        });
+        const read = vi
+            .spyOn(env.server.clients.db, 'read')
+            .mockResolvedValueOnce(
+                stale.map((h) => ({ name: h.name, source_hash: h.sourceHash })),
+            );
+        try {
+            const viaReplica = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(viaReplica[0].sourceHash).toBe(hashContent(SOURCE));
+        } finally {
+            read.mockRestore();
+        }
+        const delivery = await handlers().cachedSetForApp(cachedApp);
+        expect(delivery[0].sourceHash).toBe(hashContent(fresh));
+
+        // A publish drops the replica-read entry too.
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+            replace: true,
+        });
+        const again = await handlers().cachedSetForApp(cachedApp, {
+            missRead: 'replica',
+        });
+        expect(again[0].sourceHash).toBe(hashContent(SOURCE));
+    });
+
+    it('checks an empty replica read on the primary', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        // The replica hasn't seen the publish yet.
+        const read = vi
+            .spyOn(env.server.clients.db, 'read')
+            .mockResolvedValueOnce([]);
+        const pread = vi.spyOn(env.server.clients.db, 'pread');
+        try {
+            const set = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(set.map((h) => h.name)).toEqual(['a']);
+            expect(pread).toHaveBeenCalledTimes(1);
+
+            // The primary's answer is cached for the next run. (SQLite's
+            // `pread` goes through `read`, so count from here.)
+            const reads = read.mock.calls.length;
+            await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(read).toHaveBeenCalledTimes(reads);
+            expect(pread).toHaveBeenCalledTimes(1);
+        } finally {
+            read.mockRestore();
+            pread.mockRestore();
+        }
+    });
+
+    it('re-reads on the primary a replica set without the handler a run needs', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        const stale = await handlers().setForApp(cachedApp);
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'b',
+            source: SOURCE,
+        });
+        // The replica has only `a`, published before `b`.
+        const read = vi
+            .spyOn(env.server.clients.db, 'read')
+            .mockResolvedValueOnce(
+                stale.map((h) => ({ name: h.name, source_hash: h.sourceHash })),
+            );
+        try {
+            const set = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+                handlerName: 'b',
+            });
+            expect(set.map((h) => h.name).sort()).toEqual(['a', 'b']);
+        } finally {
+            read.mockRestore();
+        }
+    });
+
     it('follows every publish and removal', async () => {
         await handlers().publish({
             appUid: cachedApp,
@@ -495,7 +657,12 @@ describe('the set a delivery addresses', () => {
 
             expect(emit).toHaveBeenCalledWith(
                 'outer.cacheUpdate',
-                { cacheKey: [`eventHandlers:set:${cachedApp}`] },
+                {
+                    cacheKey: [
+                        `eventHandlers:set:${cachedApp}`,
+                        `eventHandlers:set:${cachedApp}:r`,
+                    ],
+                },
                 {},
             );
         } finally {
@@ -627,7 +794,11 @@ describe('listing an owner`s events workers', () => {
     it('reports one row per app, with its handler count and app details', async () => {
         const uid = await makeApp(ownerUserId);
         await handlers().publish({ appUid: uid, name: 'a', source: SOURCE });
-        await handlers().publish({ appUid: uid, name: 'b', source: OTHER_SOURCE });
+        await handlers().publish({
+            appUid: uid,
+            name: 'b',
+            source: OTHER_SOURCE,
+        });
 
         const page = await handlers().listEventsWorkersForOwner(ownerUserId);
 
@@ -654,7 +825,11 @@ describe('listing an owner`s events workers', () => {
         const uidA = await makeApp(ownerUserId);
         const uidB = await makeApp(ownerUserId);
         await handlers().publish({ appUid: uidA, name: 'a', source: SOURCE });
-        await handlers().publish({ appUid: uidB, name: 'b', source: OTHER_SOURCE });
+        await handlers().publish({
+            appUid: uidB,
+            name: 'b',
+            source: OTHER_SOURCE,
+        });
 
         const page = await handlers().listEventsWorkersForOwner(ownerUserId, {
             appUid: uidA,
@@ -678,7 +853,11 @@ describe('listing an owner`s events workers', () => {
         const uids: string[] = [];
         for (let i = 0; i < 3; i++) {
             const uid = await makeApp(ownerUserId);
-            await handlers().publish({ appUid: uid, name: 'a', source: SOURCE });
+            await handlers().publish({
+                appUid: uid,
+                name: 'a',
+                source: SOURCE,
+            });
             uids.push(uid);
         }
 
@@ -696,9 +875,7 @@ describe('listing an owner`s events workers', () => {
         expect(second.cursor).toBeUndefined();
 
         expect(
-            [...first.items, ...second.items]
-                .map((row) => row.appUid)
-                .sort(),
+            [...first.items, ...second.items].map((row) => row.appUid).sort(),
         ).toEqual([...uids].sort());
     });
 });

src/backend/stores/events/EventHandlerStore.ts

@@ -73,6 +73,13 @@ export const HANDLER_SET_CACHE_TTL_SECONDS = 30;
 const handlerSetCacheKey = (appUid: string): string =>
     `eventHandlers:set:${appUid}`;
 
+/**
+ * The same set as read from the replica, kept apart so a lagging read can never
+ * stand in for the primary's answer on the delivery path.
+ */
+const replicaSetCacheKey = (appUid: string): string =>
+    `${handlerSetCacheKey(appUid)}:r`;
+
 // -- Shapes -----------------------------------------------------------
 
 /** A row in `event_handlers`, as the rest of the system sees it. */
@@ -337,13 +344,16 @@ export class EventHandlerStore extends PuterStore {
      */
     async setForApp(
         appUid: string,
+        options: { replica?: boolean } = {},
     ): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
-        const rows = await this.clients.db.pread(
+        const sql =
             'SELECT `name`, `source_hash` ' +
-                `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
-                'ORDER BY `name` LIMIT ?',
-            [appUid, EVENTS_HANDLERS_PER_APP],
-        );
+            `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
+            'ORDER BY `name` LIMIT ?';
+        const params = [appUid, EVENTS_HANDLERS_PER_APP];
+        const rows = options.replica
+            ? await this.clients.db.read(sql, params)
+            : await this.clients.db.pread(sql, params);
         return rows.map((row) => ({
             name: String(row.name),
             sourceHash: String(row.source_hash),
@@ -354,23 +364,43 @@ export class EventHandlerStore extends PuterStore {
      * `setForApp` for the delivery path, which asks once per invocation. Cached
      * for up to {@link HANDLER_SET_CACHE_TTL_SECONDS}, and dropped everywhere by
      * every write here. An empty set is not cached: it fails the delivery, and
-     * must not outlast the publish that fills it.
+     * must not outlast the publish that fills it. Schedule runs read a miss
+     * from the replica (`missRead: 'replica'`), since they come by the clock
+     * rather than after a write, and cache it under a key of their own; an
+     * empty replica read is checked on the primary, since a lagging replica
+     * right after a publish would fail the run. Called by extensions.
      */
     async cachedSetForApp(
         appUid: string,
+        options: {
+            missRead?: 'primary' | 'replica';
+            handlerName?: string;
+        } = {},
     ): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
-        const key = handlerSetCacheKey(appUid);
+        const replica = options.missRead === 'replica';
+        // A set without the handler may be from before its publish.
+        const usable = (set: Pick<EventHandler, 'name'>[]) =>
+            set.length > 0 &&
+            (options.handlerName === undefined ||
+                set.some((handler) => handler.name === options.handlerName));
+        const key = replica
+            ? replicaSetCacheKey(appUid)
+            : handlerSetCacheKey(appUid);
         try {
             const cached = JSON.parse(
                 (await this.clients.redis.get(key)) ?? 'null',
             ) as unknown;
-            if (Array.isArray(cached) && cached.length > 0)
+            if (
+                Array.isArray(cached) &&
+                usable(cached as Pick<EventHandler, 'name'>[])
+            )
                 return cached as Pick<EventHandler, 'name' | 'sourceHash'>[];
         } catch {
             // Read the table instead.
         }
 
-        const set = await this.setForApp(appUid);
+        let set = await this.setForApp(appUid, { replica });
+        if (replica && !usable(set)) set = await this.setForApp(appUid);
         if (set.length > 0) {
             try {
                 await this.clients.redis.set(
@@ -616,7 +646,7 @@ export class EventHandlerStore extends PuterStore {
     /** Drop the delivery path's cached set here and on every peer. */
     async #forgetSet(appUid: string): Promise<void> {
         await this.publishCacheKeys({
-            keys: [handlerSetCacheKey(appUid)],
+            keys: [handlerSetCacheKey(appUid), replicaSetCacheKey(appUid)],
             broadcast: true,
         });
     }

src/backend/stores/events/KvShareHandleStore.integration.test.ts

@@ -326,7 +326,7 @@ describe('the notify flag', () => {
 });
 
 describe('live handles by owner and app', () => {
-    it('lists only the owner\'s live handles in that app, oldest first', async () => {
+    it("lists only the owner's live handles in that app, oldest first", async () => {
         const appUid = 'app-owner-listing';
         const first = await store().mint(
             input({ appUid, keyPrefix: 'owned-a:' }),
@@ -502,13 +502,11 @@ describe('the live cache across nodes', () => {
         expect(await store().getLive(row.handle)).toBeNull();
     });
 
-    it('drops an owner\'s cached rows when the owner is suspended, here and on other nodes', async () => {
+    it("drops an owner's cached rows when the owner is suspended, here and on other nodes", async () => {
         const row = await store().mint(
             input({ appUid: 'app-forget', keyPrefix: 'forget-owner:' }),
         );
-        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
-            false,
-        );
+        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
         const setSuspended = (value: number) =>
             env.server.clients.db.write(
                 'UPDATE `user` SET `suspended` = ? WHERE `id` = ?',
@@ -534,11 +532,7 @@ describe('the live cache across nodes', () => {
                     ([key]) => key === 'outer.pubsub.kv.share.forgetOwner',
                 ),
             ).toEqual([
-                [
-                    'outer.pubsub.kv.share.forgetOwner',
-                    { ownerUserId },
-                    {},
-                ],
+                ['outer.pubsub.kv.share.forgetOwner', { ownerUserId }, {}],
             ]);
 
             // Another node lifting the suspension reaches this one's cache.
@@ -557,13 +551,11 @@ describe('the live cache across nodes', () => {
         }
     });
 
-    it('drops an owner\'s cached rows when the user store suspends or unsuspends them', async () => {
+    it("drops an owner's cached rows when the user store suspends or unsuspends them", async () => {
         const row = await store().mint(
             input({ appUid: 'app-forget', keyPrefix: 'forget-store:' }),
         );
-        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
-            false,
-        );
+        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
         try {
             await env.server.stores.user.update(ownerUserId, {
                 suspended: true,
@@ -895,21 +887,27 @@ describe('retired handle retention', () => {
         expect(await exists(live)).toBe(true);
     });
 
-    it('purges once per day across nodes, keeping live and recent rows', async () => {
+    it('purges retired handles past the retention, keeping live and recent rows', async () => {
         const kvShare = env.server.services.kvShare;
-        await env.server.clients.redis.del('kvShare:purge:gate');
         const old = await retiredAt('purge-old:', now() - 31 * DAY);
         const recent = await retiredAt('purge-recent:', now() - 29 * DAY);
         const live = await retiredAt('purge-live:', null);
 
-        expect(await kvShare.purgeRevokedHandles()).toBeGreaterThanOrEqual(1);
+        expect(
+            await kvShare.purgeRevokedHandles(new AbortController().signal),
+        ).toBeGreaterThanOrEqual(1);
         expect(await exists(old)).toBe(false);
         expect(await exists(recent)).toBe(true);
         expect(await exists(live)).toBe(true);
+    });
 
-        // Another node checking the same day finds the gate held.
-        const again = await retiredAt('purge-again:', now() - 31 * DAY);
-        expect(await kvShare.purgeRevokedHandles()).toBeNull();
-        expect(await exists(again)).toBe(true);
+    it('stops purging between batches once its signal aborts', async () => {
+        const kvShare = env.server.services.kvShare;
+        const old = await retiredAt('purge-aborted:', now() - 31 * DAY);
+        const controller = new AbortController();
+        controller.abort();
+
+        expect(await kvShare.purgeRevokedHandles(controller.signal)).toBe(0);
+        expect(await exists(old)).toBe(true);
     });
 });

src/backend/stores/events/columnWidths.ts

@@ -19,8 +19,11 @@
 
 import { HttpError } from '../../core/http/HttpError.js';
 
-/** How wide one column is, and the name a caller knows the value by. */
-interface ColumnWidth {
+/**
+ * How wide one column is, and the name a caller knows the value by. Called by
+ * extensions.
+ */
+export interface ColumnWidth {
     /** Column in the migrations, so a schema change is greppable from here. */
     column: string;
     max: number;

src/backend/stores/events/types.ts

@@ -38,6 +38,20 @@ export const runsInWorker = (row: {
     handlerName?: string | null;
 }): boolean => row.appUid !== null && !!row.handlerName;
 
+/**
+ * Token, and subject, of the subscription row a recurring or `once` schedule
+ * owns. Those rows carry the schedule's lifecycle (suspend, resume, reap) and
+ * are kept out of dispatch, the durable quotas and the durable listing. Called
+ * by extensions.
+ */
+export const SCHEDULE_TOKEN_PREFIX = 'schedule:';
+
+export const scheduleToken = (scheduleUid: string): string =>
+    `${SCHEDULE_TOKEN_PREFIX}${scheduleUid}`;
+
+export const isScheduleRow = (row: { token: string }): boolean =>
+    row.token.startsWith(SCHEDULE_TOKEN_PREFIX);
+
 /**
  * What a row's delivery re-check runs against. An access mode for a filesystem
  * row, which composes with its anchor uid; the whole grant string for a row on

src/backend/stores/notification/NotificationStore.js

@@ -33,6 +33,9 @@ import { PuterStore } from '../types';
  * }} NotificationInsert
  */
 
+/** How long a probe that found no `send_key` column stands. */
+const SEND_KEY_REPROBE_MS = 60_000;
+
 /**
  * Fixed width and compared byte for byte on every engine, whatever `group`
  * holds.
@@ -75,6 +78,15 @@ const sliceWhere = (userId, { audience, appUid, excludeTypes }) => {
 };
 
 export class NotificationStore extends PuterStore {
+    #sendKeyPresent = false;
+    #sendKeyProbedAt = Number.NEGATIVE_INFINITY;
+    /** @type {Promise<void> | null} */
+    #sendKeyProbe = null;
+
+    async onServerStart() {
+        await this.#hasSendKey();
+    }
+
     // -- Reads --------------------------------------------------------
 
     /**
@@ -96,6 +108,20 @@ export class NotificationStore extends PuterStore {
         return this.#normalizeRow(rows[0]) ?? null;
     }
 
+    /**
+     * The row a delayed send wrote or last replaced, from the primary. Called
+     * by extensions.
+     *
+     * @param {string} sendKey
+     */
+    async getBySendKey(sendKey) {
+        const rows = await this.clients.db.pread(
+            'SELECT * FROM `notification` WHERE `send_key` = ? LIMIT 1',
+            [sendKey],
+        );
+        return this.#normalizeRow(rows[0]) ?? null;
+    }
+
     /**
      * The newest row `appUid` wrote for the user under `group`, or null when it
      * was shown or acknowledged: an older row is never picked up in its place.
@@ -341,6 +367,7 @@ export class NotificationStore extends PuterStore {
             type,
             group_key: key,
             replaced_at: null,
+            send_key: null,
         });
     }
 
@@ -354,6 +381,95 @@ export class NotificationStore extends PuterStore {
         return (await this.create(args)).uid;
     }
 
+    /**
+     * The statements that write a delayed send's notification, for the caller's
+     * transaction. Each runs only while `guard` holds. With `replaceUid` the
+     * open row is replaced in place and takes `sendKey`; the insert, whose uid
+     * is `sendKey`, then runs only if that row was shown or dismissed in
+     * between. Called by extensions.
+     *
+     * @param {{
+     *     sendKey: string;
+     *     userId: number;
+     *     value: unknown;
+     *     type: string;
+     *     audience: string;
+     *     appUid: string | null;
+     *     group?: string | null;
+     *     replaceUid?: string | null;
+     *     at: number;
+     * }} send
+     *   `at` is epoch ms, the row's time either way.
+     * @param {{ table: string; where: string; values: unknown[] }} guard
+     * @returns {{ statement: string; values: unknown[] }[]}
+     */
+    sendEntries(send, guard) {
+        const db = this.clients.db;
+        const serialized = JSON.stringify(send.value ?? {});
+        const seconds = Math.floor(send.at / 1000);
+        const fence = `SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where}`;
+        const entries = [];
+        if (send.replaceUid) {
+            entries.push({
+                statement:
+                    'UPDATE `notification` SET `value` = ?, `shown` = NULL, ' +
+                    `\`created_at\` = ${db.timestampFromUnixSql()}, ` +
+                    '`replaced_at` = ?, `send_key` = ? WHERE `uid` = ? ' +
+                    'AND `user_id` = ? AND `shown` IS NULL ' +
+                    `AND \`acknowledged\` IS NULL AND EXISTS (${fence})`,
+                values: [
+                    serialized,
+                    seconds,
+                    send.at,
+                    send.sendKey,
+                    send.replaceUid,
+                    send.userId,
+                    ...guard.values,
+                ],
+            });
+        }
+        // Already written: the new row's uid is the key, or a replaced row
+        // carries it. Point lookups on `uid`, since a miss on the mostly-NULL
+        // `send_key` index gap-locks where every plain insert goes.
+        const written = send.replaceUid
+            ? {
+                  sql:
+                      'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?) ' +
+                      'AND NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ? AND `send_key` = ?)',
+                  values: [send.sendKey, send.replaceUid, send.sendKey],
+              }
+            : {
+                  sql: 'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?)',
+                  values: [send.sendKey],
+              };
+        // Postgres can't type a bare parameter in a SELECT list.
+        const param = (type) =>
+            db.case({ postgres: `CAST(? AS ${type})`, otherwise: '?' });
+        entries.push({
+            statement:
+                'INSERT INTO `notification` (`uid`, `user_id`, `value`, `type`, ' +
+                '`audience`, `app_uid`, `group_key`, `created_at`, `send_key`) ' +
+                `SELECT ${param('TEXT')}, ${param('INTEGER')}, ${param('JSONB')}, ` +
+                `${param('TEXT')}, ${param('TEXT')}, ${param('TEXT')}, ` +
+                `${param('TEXT')}, ${db.timestampFromUnixSql()}, ${param('TEXT')} ` +
+                `FROM \`${guard.table}\` WHERE ${guard.where} AND ${written.sql}`,
+            values: [
+                send.sendKey,
+                send.userId,
+                serialized,
+                send.type,
+                send.audience,
+                send.appUid,
+                groupKey(send.group),
+                seconds,
+                send.sendKey,
+                ...guard.values,
+                ...written.values,
+            ],
+        });
+        return entries;
+    }
+
     /**
      * Rewrite a notification the recipient hasn't dismissed. `shown` is cleared
      * with it, so changed wording goes out again on their next connect —
@@ -362,6 +478,10 @@ export class NotificationStore extends PuterStore {
      * `onlyUnseen` also skips a row already shown. `replacedAt`, in epoch ms,
      * records the rewrite as a replacement and dates the row to it.
      *
+     * `send_key` is cleared with it whenever the column exists, on every node:
+     * the content is no longer what a delayed send wrote, so its recovery must
+     * not deliver it under that send's key.
+     *
      * False when no such row exists, so callers can fall back to a fresh
      * notification instead of dropping what they were reporting.
      *
@@ -378,6 +498,7 @@ export class NotificationStore extends PuterStore {
         const serialized =
             typeof value === 'string' ? value : JSON.stringify(value ?? {});
         const set = ['`value` = ?', '`shown` = NULL'];
+        if (await this.#hasSendKey()) set.push('`send_key` = NULL');
         const params = [serialized];
         if (replacedAt !== undefined) {
             set.push(
@@ -478,6 +599,45 @@ export class NotificationStore extends PuterStore {
 
     // -- Internals ----------------------------------------------------
 
+    /**
+     * Whether `notification.send_key` exists, asked of the primary. Present is
+     * kept for the process; absent is asked again at most once a minute, so a
+     * node started before the column was added clears it soon after.
+     *
+     * @returns {Promise<boolean>}
+     */
+    async #hasSendKey() {
+        if (this.#sendKeyPresent) return true;
+        if (Date.now() - this.#sendKeyProbedAt < SEND_KEY_REPROBE_MS)
+            return false;
+        if (this.#sendKeyProbe) {
+            await this.#sendKeyProbe;
+            return this.#sendKeyPresent;
+        }
+        const probe = this.#probeSendKey();
+        this.#sendKeyProbe = probe;
+        try {
+            await probe;
+        } finally {
+            if (this.#sendKeyProbe === probe) this.#sendKeyProbe = null;
+        }
+        return this.#sendKeyPresent;
+    }
+
+    /** Never rejects: an error reads as no column. */
+    async #probeSendKey() {
+        try {
+            await this.clients.db.pread(
+                'SELECT `send_key` FROM `notification` WHERE 1 = 0',
+                [],
+            );
+            this.#sendKeyPresent = true;
+        } catch {
+            this.#sendKeyPresent = false;
+        }
+        this.#sendKeyProbedAt = Date.now();
+    }
+
     #normalizeRow(row) {
         if (!row) return null;
         if (typeof row.value === 'string') {

src/backend/stores/notification/NotificationStore.sendKey.test.ts

@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `send_key` is cleared on every rewrite once the column exists, whichever node
+ * rewrites and whatever its config: a row a delayed send wrote and another node
+ * replaced must not be delivered again under that send's key.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { NotificationStore } from './NotificationStore.js';
+
+let server: PuterServer;
+let userId: number;
+
+/** A store on the same database, as another node with its own config. */
+const node = (config: Partial<IConfig>): NotificationStore =>
+    new NotificationStore(
+        config as IConfig,
+        server.clients as never,
+        server.stores as never,
+    );
+
+/** A row as a delayed send writes it, keyed `sendKey`. */
+const writeSent = async (store: NotificationStore): Promise<string> => {
+    const sendKey = uuidv4();
+    await server.clients.db.batchWrite(
+        store.sendEntries(
+            {
+                sendKey,
+                userId,
+                value: { title: 'later' },
+                type: 'app-message',
+                audience: 'account',
+                appUid: null,
+                at: Date.now(),
+            },
+            { table: 'user', where: '`id` = ?', values: [userId] },
+        ),
+    );
+    return sendKey;
+};
+
+const sendKeyOf = async (uid: string): Promise<unknown> => {
+    const [row] = await server.clients.db.pread(
+        'SELECT `send_key` FROM `notification` WHERE `uid` = ?',
+        [uid],
+    );
+    return row?.send_key;
+};
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    const username = `sendkey-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+    });
+    userId = user.id;
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+afterEach(() => {
+    vi.restoreAllMocks();
+});
+
+describe('clearing send_key on a rewrite', () => {
+    it('clears it from a row another node wrote, whatever either config says', async () => {
+        const writer = node({ events: { enabled: true } } as Partial<IConfig>);
+        const rewriter = node({});
+        const sendKey = await writeSent(writer);
+        expect(await sendKeyOf(sendKey)).toBe(sendKey);
+
+        await expect(
+            rewriter.updateValue(sendKey, userId, { title: 'replaced' }),
+        ).resolves.toBe(true);
+        expect(await sendKeyOf(sendKey)).toBeNull();
+    });
+
+    it('probes once and keeps a present column for the process', async () => {
+        const store = node({});
+        const pread = vi.spyOn(server.clients.db, 'pread');
+        const first = await writeSent(store);
+        const second = await writeSent(store);
+
+        await store.updateValue(first, userId, { v: 1 });
+        await store.updateValue(second, userId, { v: 2 });
+
+        const probes = pread.mock.calls.filter(([sql]) =>
+            String(sql).includes('WHERE 1 = 0'),
+        );
+        expect(probes).toHaveLength(1);
+        expect(await sendKeyOf(first)).toBeNull();
+        expect(await sendKeyOf(second)).toBeNull();
+    });
+
+    it('rewrites without naming the column while it is absent, and clears it after the next probe once it exists', async () => {
+        const db = server.clients.db;
+        const store = node({});
+        const sendKey = await writeSent(store);
+        const write = vi.spyOn(db, 'write');
+        const realPread = db.pread.bind(db);
+        let columnExists = false;
+        vi.spyOn(db, 'pread').mockImplementation(async (sql, values) => {
+            if (!columnExists && String(sql).includes('WHERE 1 = 0'))
+                throw new Error('no such column: send_key');
+            return realPread(sql, values);
+        });
+        let now = Date.now();
+        vi.spyOn(Date, 'now').mockImplementation(() => now);
+        const rewriteSql = () =>
+            String(
+                write.mock.calls.findLast(([sql]) =>
+                    String(sql).startsWith('UPDATE `notification` SET `value`'),
+                )?.[0],
+            );
+
+        await store.updateValue(sendKey, userId, { v: 1 });
+        expect(rewriteSql()).not.toContain('send_key');
+        expect(await sendKeyOf(sendKey)).toBe(sendKey);
+
+        // Within the minute the absent answer stands.
+        columnExists = true;
+        now += 30_000;
+        await store.updateValue(sendKey, userId, { v: 2 });
+        expect(rewriteSql()).not.toContain('send_key');
+        expect(await sendKeyOf(sendKey)).toBe(sendKey);
+
+        now += 31_000;
+        await store.updateValue(sendKey, userId, { v: 3 });
+        expect(rewriteSql()).toContain('`send_key` = NULL');
+        expect(await sendKeyOf(sendKey)).toBeNull();
+    });
+
+    it('shares one probe between rewrites that race it', async () => {
+        const db = server.clients.db;
+        const store = node({});
+        const keys = await Promise.all([writeSent(store), writeSent(store)]);
+        const pread = vi.spyOn(db, 'pread');
+
+        await Promise.all(
+            keys.map((uid) => store.updateValue(uid, userId, { v: 1 })),
+        );
+
+        const probes = pread.mock.calls.filter(([sql]) =>
+            String(sql).includes('WHERE 1 = 0'),
+        );
+        expect(probes).toHaveLength(1);
+        for (const uid of keys) expect(await sendKeyOf(uid)).toBeNull();
+    });
+});
+
+describe('writing a delayed send twice', () => {
+    const entriesFor = (sendKey: string, replaceUid: string | null = null) =>
+        node({}).sendEntries(
+            {
+                sendKey,
+                userId,
+                value: { title: 'later' },
+                type: 'app-message',
+                audience: 'account',
+                appUid: null,
+                replaceUid,
+                at: Date.now(),
+            },
+            { table: 'user', where: '`id` = ?', values: [userId] },
+        );
+    const rowsKeyed = async (uid: string): Promise<number> =>
+        (
+            await server.clients.db.pread(
+                'SELECT 1 FROM `notification` WHERE `uid` = ?',
+                [uid],
+            )
+        ).length;
+
+    it('writes nothing again once the row was rewritten and lost its key', async () => {
+        const sendKey = await writeSent(node({}));
+        await node({}).updateValue(sendKey, userId, { title: 'replaced' });
+        expect(await sendKeyOf(sendKey)).toBeNull();
+
+        await server.clients.db.batchWrite(entriesFor(sendKey));
+        expect(await rowsKeyed(sendKey)).toBe(1);
+    });
+
+    it('adds no row once the replaced row carries the key', async () => {
+        const replaced = await writeSent(node({}));
+        const sendKey = uuidv4();
+        await server.clients.db.batchWrite(entriesFor(sendKey, replaced));
+        expect(await sendKeyOf(replaced)).toBe(sendKey);
+        expect(await rowsKeyed(sendKey)).toBe(0);
+
+        await server.clients.db.batchWrite(entriesFor(sendKey, replaced));
+        expect(await rowsKeyed(sendKey)).toBe(0);
+    });
+});

src/backend/stores/notification/NotificationStore.test.js

@@ -346,7 +346,9 @@ describe('NotificationStore', () => {
         const uids = (rows) => rows.map((r) => r.uid).sort();
 
         expect(
-            uids(await store.listScoped(u.id, { audience: 'developer', appUid })),
+            uids(
+                await store.listScoped(u.id, { audience: 'developer', appUid }),
+            ),
         ).toEqual([mine.uid]);
         // `null` is the rows naming no app, not "any app".
         expect(

src/backend/stores/session/SessionStore.js

@@ -88,6 +88,8 @@ const predates = (row, notBefore) =>
     Number(row.created_at) < notBefore;
 
 export class SessionStore extends PuterStore {
+    /** Worker lookups under way, so a burst of first calls inserts once. */
+    #workerLookups = new Map();
     #lastSessionTouchMs = new Map();
     #lastUserTouchMs = new Map();
 
@@ -847,7 +849,21 @@ export class SessionStore extends PuterStore {
      */
     async getOrCreateWorker(userId, opts = {}) {
         if (!userId || !opts.workerName) return null;
+        const key = `${this.#cacheKeyWorker(
+            userId,
+            opts.appUid ?? null,
+            String(opts.workerName),
+        )}|${opts.notBefore ?? ''}`;
+        const running = this.#workerLookups.get(key);
+        if (running) return running;
+        const started = this.#getOrCreateWorker(userId, opts).finally(() =>
+            this.#workerLookups.delete(key),
+        );
+        this.#workerLookups.set(key, started);
+        return started;
+    }
 
+    async #getOrCreateWorker(userId, opts) {
         const existing = await this.getWorker(userId, opts);
         if (existing && !predates(existing, opts.notBefore ?? null)) {
             return existing;

src/backend/stores/session/SessionStore.test.ts

@@ -1008,6 +1008,7 @@ describe('SessionStore', () => {
             const user = await makeUser();
             const appUid = `app-${uuidv4()}`;
             const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
+            const write = vi.spyOn(server.clients.db, 'write');
             const results = await Promise.all(
                 Array.from({ length: 10 }, () =>
                     target.getOrCreateWorker(user.id, { appUid, workerName }),
@@ -1015,6 +1016,13 @@ describe('SessionStore', () => {
             );
             const uuids = new Set(results.map((r: { uuid: string }) => r.uuid));
             expect(uuids.size).toBe(1);
+            // One process inserts once, however many ask at the same time.
+            expect(
+                write.mock.calls.filter(([sql]) =>
+                    String(sql).startsWith('INSERT INTO `sessions`'),
+                ),
+            ).toHaveLength(1);
+            write.mockRestore();
 
             // And only one row ever made it into the table.
             const rows = await server.clients.db.read(
@@ -1024,6 +1032,62 @@ describe('SessionStore', () => {
             expect(rows).toHaveLength(1);
         });
 
+        it('a failed first insert rejects every racer, and the next call starts afresh', async () => {
+            const user = await makeUser();
+            const appUid = `app-${uuidv4()}`;
+            const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
+            const write = vi
+                .spyOn(server.clients.db, 'write')
+                .mockImplementationOnce(async () => {
+                    throw new Error('primary down');
+                });
+            const racers = Array.from({ length: 3 }, () =>
+                target.getOrCreateWorker(user.id, { appUid, workerName }),
+            );
+            for (const racer of racers)
+                await expect(racer).rejects.toThrow('primary down');
+            write.mockRestore();
+            const row = await target.getOrCreateWorker(user.id, {
+                appUid,
+                workerName,
+            });
+            expect(row.kind).toBe('worker');
+        });
+
+        it('lookups for different apps do not wait on each other', async () => {
+            const user = await makeUser();
+            const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
+            const db = server.clients.db;
+            const original = db.write.bind(db);
+            let release!: () => void;
+            const held = new Promise<void>((resolve) => (release = resolve));
+            // The first app's insert hangs until released.
+            const write = vi
+                .spyOn(db, 'write')
+                .mockImplementationOnce(async (...args) => {
+                    await held;
+                    return original(...(args as [string, unknown[]]));
+                });
+            const slow = target.getOrCreateWorker(user.id, {
+                appUid: `app-${uuidv4()}`,
+                workerName,
+            });
+            const other = await Promise.race([
+                target.getOrCreateWorker(user.id, {
+                    appUid: `app-${uuidv4()}`,
+                    workerName,
+                }),
+                new Promise((resolve) =>
+                    setTimeout(() => resolve('stuck'), 1500),
+                ),
+            ]);
+            expect(other).not.toBe('stuck');
+            release();
+            const slowRow = await slow;
+            expect(slowRow.uuid).not.toBe((other as { uuid: string }).uuid);
+            write.mockRestore();
+        });
+
         it('returns null when called with falsy inputs', async () => {
             expect(
                 await target.getOrCreateWorker(null, { workerName: 'wk' }),
@@ -1094,8 +1158,9 @@ describe('SessionStore', () => {
             const uuids = rows.map((row: { uuid: string }) => row.uuid);
             expect(uuids).toContain(a.uuid);
             expect(uuids).toContain(b.uuid);
-            expect(rows.every((row: { id: number }) => Number.isFinite(row.id)))
-                .toBe(true);
+            expect(
+                rows.every((row: { id: number }) => Number.isFinite(row.id)),
+            ).toBe(true);
         });
 
         it('does not return a revoked session', async () => {

src/backend/testUtil.ts

@@ -152,6 +152,8 @@ export const setupTestServer = async (
             database: testDatabaseDefault(),
             dynamo: { inMemory: true, bootstrapTables: true },
             redis: { useMock: true },
+            // Suites run system jobs themselves, not on the poll timer.
+            jobs: { enabled: false },
             s3: { localConfig: { inMemory: true } },
             no_default_user: true,
             no_devwatch: true,

src/backend/types.ts

@@ -256,7 +256,12 @@ export interface IPreludeConfig {
      * an RCS agent provisioned in the Prelude account to actually use RCS.
      */
     preferredChannel?:
-        'sms' | 'rcs' | 'whatsapp' | 'viber' | 'zalo' | 'telegram';
+        | 'sms'
+        | 'rcs'
+        | 'whatsapp'
+        | 'viber'
+        | 'zalo'
+        | 'telegram';
 }
 
 /**
@@ -642,6 +647,21 @@ export interface IDevWatcherConfig {
     }>;
 }
 
+/**
+ * System jobs config (`jobs`). An extension that runs jobs types its own keys
+ * by augmenting this interface.
+ *
+ * - `namespace` — keys this deployment's jobs and schedules, so deployments
+ *   sharing one database or Redis never run each other's. Defaults to `domain`;
+ *   one that can't be held falls back to `default` (`resolveJobsNamespace`).
+ * - `enabled` — `false` keeps this node from polling for jobs (logged at start);
+ *   the rest of the fleet still runs them. Test servers turn it off.
+ */
+export interface IJobsConfig {
+    namespace?: string;
+    enabled?: boolean;
+}
+
 /**
  * Root config. Only `port` and `extensions` are mandatory; they are added in
  * `IConfig` below. Every config field is declared here with a doc comment so
@@ -1270,6 +1290,16 @@ interface IConfigOptional {
         forwardSession?: boolean;
         /** How long a handler has to answer an invocation. Default 30 s. */
         invokeTimeoutMs?: number;
+        /**
+         * Invocations sent to the dispatcher in one request. Default 60; 1
+         * sends each on its own.
+         */
+        invokeBatchSize?: number;
+        /**
+         * How long the first invocation of a batch waits for others to join it.
+         * Default 5 ms.
+         */
+        invokeBatchWindowMs?: number;
         /**
          * Dispatch namespace events workers are deployed into. Must not be the
          * one the public worker dispatcher serves — that namespace is reachable
@@ -1291,6 +1321,9 @@ interface IConfigOptional {
         internalSecret?: string;
     };
 
+    /** System jobs (`services.jobs`); see {@link IJobsConfig}. */
+    jobs?: IJobsConfig;
+
     /**
      * Key-value sharing beyond watching.
      *
@@ -1346,8 +1379,7 @@ export interface ServerDrainInfo {
 }
 
 export interface WithCostsReporting extends WithLifecycle {
-    getReportedCosts?: () =>
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    getReportedCosts?: () => // eslint-disable-next-line @typescript-eslint/no-explicit-any
         | Promise<Record<string, any>[]>
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         | Record<string, any>[];

src/backend/util/jobsNamespace.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import type { IConfig } from '../types.js';
+import { resolveJobsNamespace } from './jobsNamespace.js';
+
+const config = (partial: Partial<IConfig>): IConfig => partial as IConfig;
+
+describe('resolveJobsNamespace', () => {
+    afterEach(() => {
+        vi.restoreAllMocks();
+    });
+
+    it('takes jobs.namespace first, then the domain, then "default"', () => {
+        expect(
+            resolveJobsNamespace(
+                config({ jobs: { namespace: 'ns-1' }, domain: 'puter.test' }),
+            ),
+        ).toBe('ns-1');
+        expect(resolveJobsNamespace(config({ domain: 'puter.test' }))).toBe(
+            'puter.test',
+        );
+        expect(resolveJobsNamespace(config({}))).toBe('default');
+    });
+
+    it('falls back to "default" for a domain with a port, logging once', () => {
+        const error = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const withPort = config({ domain: 'puter.localhost:4100' });
+        expect(resolveJobsNamespace(withPort)).toBe('default');
+        expect(resolveJobsNamespace(withPort)).toBe('default');
+        expect(error).toHaveBeenCalledTimes(1);
+        expect(String(error.mock.calls[0][0])).toContain(
+            'puter.localhost:4100',
+        );
+    });
+
+    it('falls back to "default" for an invalid jobs.namespace', () => {
+        const error = vi.spyOn(console, 'error').mockImplementation(() => {});
+        expect(
+            resolveJobsNamespace(
+                config({ jobs: { namespace: 'a:b' }, domain: 'puter.test' }),
+            ),
+        ).toBe('default');
+        expect(resolveJobsNamespace(config({ jobs: { namespace: '' } }))).toBe(
+            'default',
+        );
+        expect(
+            resolveJobsNamespace(
+                config({ jobs: { namespace: 'x'.repeat(129) } }),
+            ),
+        ).toBe('default');
+        expect(error).toHaveBeenCalledTimes(3);
+    });
+});

src/backend/util/jobsNamespace.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type { IConfig } from '../types.js';
+
+/** A namespace must fit the jobs and schedules tables and a Redis key segment. */
+const JOBS_NAMESPACE_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+const warned = new Set<string>();
+
+/**
+ * The namespace system jobs and schedules are keyed by, so deployments sharing
+ * a database or Redis stay apart: `jobs.namespace`, else `domain`, else
+ * `default`. One that can't be held (a domain with a port, say) falls back to
+ * `default`, with one logged error per value.
+ */
+export const resolveJobsNamespace = (config: IConfig): string => {
+    const configured = config.jobs?.namespace ?? config.domain ?? 'default';
+    if (JOBS_NAMESPACE_PATTERN.test(configured)) return configured;
+    if (!warned.has(configured)) {
+        warned.add(configured);
+        console.error(
+            `[jobs] namespace "${configured}" (jobs.namespace, else domain) must match ${JOBS_NAMESPACE_PATTERN}; using "default". Set jobs.namespace to keep deployments sharing a database apart.`,
+        );
+    }
+    return 'default';
+};

src/backend/util/retryBackoff.test.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from './retryBackoff.js';
+
+describe('deliveryBackoffMs', () => {
+    it('doubles from the base per failure and stops at the ceiling', () => {
+        expect(deliveryBackoffMs(0)).toBe(EVENTS_RETRY_BASE_MS);
+        expect(deliveryBackoffMs(1)).toBe(EVENTS_RETRY_BASE_MS);
+        expect(deliveryBackoffMs(3)).toBe(EVENTS_RETRY_BASE_MS * 4);
+        expect(deliveryBackoffMs(50)).toBe(EVENTS_RETRY_MAX_MS);
+    });
+});

src/backend/util/retryBackoff.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// A retry on a fixed cadence turns one broken dependency into a permanent load
+// on it, so the wait doubles per attempt up to a ceiling.
+
+/** Wait before the first retry. */
+export const EVENTS_RETRY_BASE_MS = 2_000;
+
+/** Longest wait between retries, however many have failed. */
+export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+
+/** How long to wait after `attempts` failures in a row. */
+export const deliveryBackoffMs = (attempts: number): number =>
+    Math.min(
+        EVENTS_RETRY_MAX_MS,
+        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
+    );

src/backend/util/secureHttp.test.ts

@@ -442,6 +442,46 @@ describe('secureFetch transport behaviour', () => {
             });
         });
 
+        it('follows redirects up to maxRedirects, each through the proxy', async () => {
+            respond = (res) => {
+                if (lastRequest.url === '/?url=https://upstream.test/start') {
+                    res.writeHead(301, { location: '/landing' });
+                    res.end();
+                    return;
+                }
+                res.writeHead(200);
+                res.end('landed');
+            };
+
+            const res = await withProxy(() =>
+                secureFetch('https://upstream.test/start', { maxRedirects: 1 }),
+            );
+            expect(await res.text()).toBe('landed');
+            expect(lastRequest.url).toBe('/?url=https://upstream.test/landing');
+        });
+
+        it('re-validates each hop and stops at maxRedirects', async () => {
+            respond = (res) => {
+                res.writeHead(302, { location: 'http://169.254.169.254/' });
+                res.end();
+            };
+            await withProxy(async () => {
+                await expect(
+                    secureFetch('https://upstream.test/a', { maxRedirects: 3 }),
+                ).rejects.toThrow('IP-addressed URLs are not allowed');
+            });
+
+            respond = (res) => {
+                res.writeHead(302, { location: 'https://upstream.test/loop' });
+                res.end();
+            };
+            await withProxy(async () => {
+                await expect(
+                    secureFetch('https://upstream.test/a', { maxRedirects: 2 }),
+                ).rejects.toThrow('redirects are not allowed');
+            });
+        });
+
         it('names the target as unknown when a 3xx carries no Location', async () => {
             respond = (res) => {
                 res.writeHead(304);

src/backend/util/secureHttp.ts

@@ -172,15 +172,21 @@ function proxyConfig(): ISecureCorsProxyConfig | undefined {
 interface SecureFetchInit extends Omit<RequestInit, 'redirect'> {
     /** Bypass the CORS proxy even if one is configured. Internal-only calls. */
     skipProxy?: boolean;
+    /**
+     * Follow up to this many http(s) redirects (default 0). Every hop goes
+     * through the same guards as the first request.
+     */
+    maxRedirects?: number;
 }
 
 /**
  * Fetch `url` with SSRF guards: • Rejects raw-IP / localhost hosts via
- * {@link validateUrlNoIP}. • Forces `redirect: 'manual'` and rejects any 3xx
- * response so a permissive target can't bounce us onto an internal endpoint. •
- * Re-checks DNS at connect time: every address the socket actually connects to
- * must pass {@link isPublicResolvedAddress}, so a hostname resolving to a
- * private/reserved address (including via DNS rebinding) is rejected. • If
+ * {@link validateUrlNoIP}. • Rejects any 3xx response so a permissive target
+ * can't bounce us onto an internal endpoint, unless `maxRedirects` allows it;
+ * each followed hop is re-validated. • Re-checks DNS at connect time: every
+ * address the socket actually connects to must pass
+ * {@link isPublicResolvedAddress}, so a hostname resolving to a private/reserved
+ * address (including via DNS rebinding) is rejected. • If
  * `config.secureCorsProxy` is set AND the URL isn't a `data:` URI, rewrites the
  * request through the configured signed Cloudflare Worker proxy (adds
  * `x-cors-proxy-auth-secret`).
@@ -192,6 +198,33 @@ interface SecureFetchInit extends Omit<RequestInit, 'redirect'> {
 export async function secureFetch(
     url: string,
     init: SecureFetchInit = {},
+): Promise<Response> {
+    const { maxRedirects = 0, ...rest } = init;
+    let current = url;
+    for (let hop = 0; ; hop++) {
+        const response = await secureFetchOnce(current, rest);
+        if (response.status < 300 || response.status >= 400) return response;
+        const location = response.headers.get('location');
+        const next = location ? new URL(location, current) : null;
+        if (
+            hop >= maxRedirects ||
+            !next ||
+            (next.protocol !== 'https:' && next.protocol !== 'http:')
+        ) {
+            throw new HttpError(
+                400,
+                `redirects are not allowed (target: ${location ?? 'unknown'})`,
+                { legacyCode: 'bad_request' },
+            );
+        }
+        await response.body?.cancel();
+        current = next.href;
+    }
+}
+
+async function secureFetchOnce(
+    url: string,
+    init: Omit<SecureFetchInit, 'maxRedirects'>,
 ): Promise<Response> {
     // Data URIs bypass everything — nothing to resolve, nothing to proxy.
     if (url.startsWith('data:')) {
@@ -222,20 +255,10 @@ export async function secureFetch(
     const { skipProxy: _skip, ...rest } = init;
     // lib.dom and undici fetch types are structurally equivalent here but
     // nominally distinct; cast across the boundary.
-    const response = (await undiciFetch(finalUrl, {
+    return (await undiciFetch(finalUrl, {
         ...rest,
         headers,
         redirect: 'manual',
         dispatcher: proxied ? proxyDispatcher : ssrfGuardDispatcher,
     } as unknown as Parameters<typeof undiciFetch>[1])) as unknown as Response;
-
-    if (response.status >= 300 && response.status < 400) {
-        throw new HttpError(
-            400,
-            `redirects are not allowed (target: ${response.headers.get('location') ?? 'unknown'})`,
-            { legacyCode: 'bad_request' },
-        );
-    }
-
-    return response;
 }

src/backend/util/userDetails.ts

@@ -28,6 +28,7 @@ import {
     type CardFallbackClients,
 } from './cardFallback.js';
 import { getTaskbarItems } from './taskbarItems.js';
+import { isEmailGateLast } from './verificationOrder.js';
 
 type TaskbarDeps = Parameters<typeof getTaskbarItems>[1];
 
@@ -153,6 +154,8 @@ export async function buildUserDetails(
         // client reads it. Only the verification flag ships.
         requires_phone_verification: user.requires_phone_verification,
         requires_card_verification: user.requires_card_verification,
+        // Card and phone come before email for this account.
+        email_gate_last: isEmailGateLast(user),
         // A seat reaches nothing until it replaces its admin's password.
         requires_password_change: user.requires_password_change,
         // The SMS-to-card escape hatch. It ships here because by the time the

src/backend/util/verificationOrder.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Whether the account clears card and phone verification before confirming its
+ * email, and is only sent the confirmation email once it gets there. Decided by
+ * the signup gate and recorded in the account's metadata.
+ */
+export const isEmailGateLast = (
+    user: { metadata?: Record<string, unknown> | null } | null | undefined,
+): boolean => user?.metadata?.emailGateLast === true;

src/docs/src/Auth/signOut.md

@@ -6,6 +6,8 @@ platforms: [websites, apps]
 
 Signs the user out of the application.
 
+On a website, signing out also turns push notifications off in this browser. It ends the push subscription Puter made for your app here and removes it from Puter, and [`onPush()`](/Notifications/onPush/) doesn't subscribe again until the next user calls [`enable()`](/Notifications/enable/). The user is signed out as soon as you call it. The returned promise resolves once the subscription is ended, so wait for it before you reload the page.
+
 ## Syntax
 
 ```js
@@ -18,7 +20,7 @@ None
 
 ## Return value
 
-None
+A `Promise` that resolves with nothing once the user is signed out and, on a website, this browser's push subscription for your app is ended. It doesn't reject.
 
 ## Example
 

src/docs/src/Events.md

@@ -25,7 +25,17 @@ What you watch is written as a subject string:
 
 [Subjects and events](/Events/subjects/) has the full subject syntax and the shape of each event.
 
-With the [User-Pays Model](/user-pays-model/), deliveries are billed to the user holding the subscription, not to you. See [Rate Limits and Quotas](/rate-limits-and-quotas/#events) for limits and costs.
+To run a handler on a timer instead of when data changes, create a schedule with [`onSchedule()`](/Events/onSchedule/). It runs your handler at an interval, on a cron expression in the user's time zone, or once at a set time, even while your app is closed:
+
+```js
+await puter.events.onSchedule({
+    cron: '0 9 * * 1-5',
+    tz: 'America/New_York',
+    handlerName: 'sendDigest',
+});
+```
+
+With the [User-Pays Model](/user-pays-model/), deliveries and schedule runs are billed to the user holding the subscription or schedule, not to you. See [Rate Limits and Quotas](/rate-limits-and-quotas/#events) for limits and costs.
 
 ## Features
 
@@ -247,6 +257,10 @@ These Events features are supported out of the box when using Puter.js:
 - **[`puter.events.list()`](/Events/list/)** - List the persistent subscriptions you hold
 - **[`puter.events.update()`](/Events/update/)** - Change a persistent subscription's context without ending it
 - **[`puter.events.unsubscribe()`](/Events/unsubscribe/)** - End a persistent subscription
+- **[`puter.events.onSchedule()`](/Events/onSchedule/)** - Run a handler at an interval, on a cron expression, or once at a set time
+- **[`puter.events.listSchedules()`](/Events/listSchedules/)** - List the schedules you hold
+- **[`puter.events.getSchedule()`](/Events/getSchedule/)** - Read one schedule
+- **[`puter.events.removeSchedule()`](/Events/removeSchedule/)** - End a schedule
 - **[`puter.events.fetch()`](/Events/fetch/)** - Read the notifications you missed while nothing was listening
 - **[`puter.events.handlers`](/Events/handlers/)** - Publish, list and remove the named handlers persistent subscriptions run
 - **[`puter.events.workers`](/Events/workers/)** - List and destroy an app's events worker
@@ -266,6 +280,10 @@ You can see various Puter.js Events features in action from the following exampl
   - [Find Suspended Subscriptions](/playground/events-list-suspended/)
   - [Change a Subscription's Context](/playground/events-update/)
   - [End a Persistent Subscription](/playground/events-unsubscribe/)
+- Schedules
+  - [List Schedules](/playground/events-list-schedules/)
+  - [Read a Schedule](/playground/events-get-schedule/)
+  - [Remove a Schedule](/playground/events-remove-schedule/)
 - Catching up
   - [Catch Up on Notifications](/playground/events-fetch/)
   - [Catch Up, Then Keep Listening](/playground/events-fetch-then-listen/)

src/docs/src/Events/getSchedule.md

@@ -0,0 +1,67 @@
+---
+title: puter.events.getSchedule()
+description: Read one schedule, including one that has ended.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+Reads one schedule created with [`puter.events.onSchedule()`](/Events/onSchedule/), or one notification your app stored to send later with [`puter.notifications.notify()`](/Notifications/notify/#send-it-later). An ended schedule can be read for 7 days after it ends.
+
+An app can read only the schedules it created. An account session can read any of the user's schedules. An id this caller can't read reads as not existing, so the call doesn't reveal which schedules exist.
+
+On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.getSchedule(scheduleId)
+```
+
+## Parameters
+
+#### `scheduleId` (String) (required)
+The `scheduleId` from [`onSchedule()`](/Events/onSchedule/), [`notify()`](/Notifications/notify/#send-it-later) or [`listSchedules()`](/Events/listSchedules/).
+
+## Return value
+
+A `Promise` that resolves to the schedule, the object [`onSchedule()`](/Events/onSchedule/#return-value) returns, without `off()`. Its `state` says whether it's `active`, `suspended`, `sending` or `ended`, and `nextRunAt` says when it's next due.
+
+The promise rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `events_schedule_not_found` | No such schedule, or it isn't this caller's. Also for an empty `scheduleId`. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `too_many_requests` | Over the listing rate limit, which is shared with [`listSchedules()`](/Events/listSchedules/). |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">Check when a stored notification goes out</strong>
+
+```html;events-get-schedule
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            const later = await puter.notifications.notify({
+                title: 'Stand-up in 10 minutes',
+                sendAt: Date.now() + 10 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            const schedule = await puter.events.getSchedule(later.scheduleId);
+            const due = schedule.nextRunAt ? new Date(schedule.nextRunAt).toLocaleTimeString() : 'nothing due';
+            puter.print(`${schedule.state}, due ${due}<br>`);
+
+            // Ended schedules can still be read
+            await puter.notifications.cancel(later.scheduleId);
+            const ended = await puter.events.getSchedule(later.scheduleId);
+            puter.print(`${ended.state} (${ended.endReason})<br>`);
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Events/handlers.md

@@ -1,12 +1,12 @@
 ---
 title: puter.events.handlers
-description: Publish, list and remove the named handlers a persistent subscription runs.
+description: Publish, list and remove the named handlers that persistent subscriptions and schedules run.
 platforms: [websites, apps, nodejs, workers]
 ---
 
 <div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
 
-A **handler** is a function your app publishes once, under a name, for persistent subscriptions to run. A handler runs only when a subscription bound to its name has a delivery. Nothing else can trigger it.
+A **handler** is a function your app publishes once, under a name, for [persistent subscriptions](/Events/onPersistent/) and [schedules](/Events/onSchedule/) to run. A handler runs only when a subscription bound to its name has a delivery, or a schedule bound to it is due. Nothing else can trigger it.
 
 Only the app's owner can publish, list, and remove its handlers. Inside an app, including a website using Puter.js, these calls act on that app, and work only while the signed-in user owns it. Outside an app, such as in Node.js with your account token, name the app with `appUid`.
 
@@ -40,6 +40,17 @@ await puter.events.onPersistent({ subject: 'fs:~/inbox', handlerName: 'ingestUpl
 
 An identifier the check can't resolve is refused when you publish, instead of failing on the first delivery.
 
+## What a handler is called with
+
+A handler gets one object. Which `event` it carries depends on what ran it:
+
+- **A subscription delivery** carries the change that happened, in the shape [Subjects and events](/Events/subjects/#the-event) describes, or a [gap marker](/Events/subjects/#gaps).
+- **A schedule run** carries `op: 'tick'`, the `scheduleId`, and the `scheduledFor` time the run is for, as [`onSchedule()`](/Events/onSchedule/#the-event) describes.
+
+The other bindings are the same for both: `ctx`, `user` and `fetch`, plus `ack` on a `queue` subscription. See [Where the handler runs](/Events/onPersistent/#where-the-handler-runs). One handler can serve both, and tell them apart by `event.op`.
+
+Schedule runs and `queue` deliveries are at-least-once. A retry, or a handler still running past its 30 seconds when the retry starts, can run the same event twice, and even at the same time. Both attempts have the same `event.id`, so record it once the work is done and skip ids you've already handled.
+
 ## `publish()`
 
 ```js
@@ -78,7 +89,7 @@ puter.events.handlers.list()
 puter.events.handlers.list(options)
 ```
 
-Resolves to `[{ name, hash, updatedAt, subscriptions }]` for every handler the app has published, sorted by name. Each `subscriptions` count includes the suspended subscriptions bound to that name. The `options.appUid` option works as it does for `publish()`.
+Resolves to `[{ name, hash, updatedAt, subscriptions }]` for every handler the app has published, sorted by name. Each `subscriptions` count includes the subscriptions and schedules bound to that name, suspended ones included. The `options.appUid` option works as it does for `publish()`.
 
 The source is never returned.
 
@@ -93,12 +104,12 @@ Resolves to `{ name, removed, suspended }`, where `removed` is `false` when noth
 
 | Situation | What happens |
 | --- | --- |
-| No subscriptions use the name | The handler is deleted. |
-| Subscriptions use it | The handler is deleted, and its subscriptions are suspended with `suspendedReason: 'handler_not_found'`, not deleted. The app's developer gets an `app.events.suspended` notification. |
+| No subscriptions or schedules use the name | The handler is deleted. |
+| Subscriptions or schedules use it | The handler is deleted, and its subscriptions and schedules are suspended with `suspendedReason: 'handler_not_found'`, not deleted. The app's developer gets an `app.events.suspended` notification. |
 
-Publishing the name again resumes them, with the same ids and context. This is how you recover from a bad deploy.
+Publishing the name again resumes them, with the same ids and context. This is how you recover from a bad deploy. A resumed schedule runs at its next slot, and doesn't make up the runs it missed.
 
-To rename a handler, publish the new name and remove the old one. Subscriptions don't follow the rename. Users have to subscribe again, so their subscriptions never switch to different code without them knowing.
+To rename a handler, publish the new name and remove the old one. Subscriptions and schedules don't follow the rename. Users have to subscribe again, so their subscriptions never switch to different code without them knowing.
 
 An app's first published handler creates its [events worker](/Events/workers/), and removing the last one takes it down.
 

src/docs/src/Events/listSchedules.md

@@ -0,0 +1,88 @@
+---
+title: puter.events.listSchedules()
+description: List the schedules this caller holds, one page at a time.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+Lists the schedules created with [`puter.events.onSchedule()`](/Events/onSchedule/), and the notifications your app stored to send later with [`puter.notifications.notify()`](/Notifications/notify/#send-it-later). Ended schedules stay in the list for 7 days, with the reason they ended.
+
+An app sees only the schedules it created for the user it runs for. An account session, such as the Puter desktop, sees all of the user's schedules, including ones left by apps they no longer use, so that's where to clean up stray schedules. A scoped access token, such as the one in a [`getReadURL()`](/FS/getReadURL/) URL, sees none.
+
+Schedules aren't listed by [`puter.events.list()`](/Events/list/), which lists persistent subscriptions only.
+
+On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.listSchedules()
+puter.events.listSchedules(options)
+```
+
+## Parameters
+
+#### `options` (Object) (optional)
+
+- `limit` (Number): Maximum schedules per page. Capped at 200, and defaults to 50.
+- `cursor` (String | null): The `cursor` from the previous page. Leave it out, or pass `null`, for the first page.
+- `includeTotal` (Boolean): Adds `total`, the number of schedules across every page. Ask for it on the first page only.
+
+## Return value
+
+A `Promise` for one page, `{ items, cursor?, total? }`:
+
+- `items` (Array): The schedules on this page, oldest first. Each one is the object [`onSchedule()`](/Events/onSchedule/#return-value) returns, without `off()`.
+- `cursor` (String): Present while there are more pages to load. Pass it as `cursor` to get the next page.
+- `total` (Number): The number of schedules across every page, only with `includeTotal`.
+
+Pages can be shorter than `limit` before the last one, so keep going until `cursor` is absent.
+
+The `context` a schedule was created with is never returned, since it often holds secrets. The `contextKeys` field lists its key names, and `contextHash` changes whenever a value does.
+
+The promise rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `invalid_request` | `includeTotal` isn't a boolean. |
+| `bad_request` | `limit` isn't a positive number. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `too_many_requests` | Over the listing rate limit, which is shared with [`getSchedule()`](/Events/getSchedule/) and the subscription listings. |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">List every schedule, page by page</strong>
+
+```html;events-list-schedules
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            // (1) Something to list: a notification to send in an hour
+            const later = await puter.notifications.notify({
+                title: 'Time for a break',
+                sendAt: Date.now() + 60 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            // (2) Page through everything this app holds for the user
+            let cursor = null;
+            do {
+                const page = await puter.events.listSchedules({ cursor, limit: 50 });
+                for (const schedule of page.items) {
+                    const next = schedule.nextRunAt ? new Date(schedule.nextRunAt).toLocaleString() : 'nothing due';
+                    puter.print(`${schedule.kind} ${schedule.state}, next: ${next}<br>`);
+                }
+                cursor = page.cursor ?? null;
+            } while (cursor);
+
+            // (3) Cleanup
+            await puter.notifications.cancel(later.scheduleId);
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Events/onSchedule.md

@@ -0,0 +1,316 @@
+---
+title: puter.events.onSchedule()
+description: Run your app's handler at an interval, on a cron expression in a time zone, or once at a set time, even while your app is closed.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+<div class="info">Schedules are available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+Runs a handler your app published with [`puter.events.handlers.publish()`](/Events/handlers/) on a timer. A schedule can run at an interval, on a cron expression in a time zone, or once at a set time. It's stored on the user's account and runs in your app's [events worker](/Events/workers/), whether or not your app is open. It never runs in the client that created it.
+
+To run a handler when the user's data changes instead, use [`puter.events.onPersistent()`](/Events/onPersistent/).
+
+[Change Data at a Set Time](/recipes/change-data-at-a-set-time/) walks through a one-time schedule with `at`, including how to cancel and move it.
+
+Only an app can create a schedule, meaning an app running on Puter or a website using Puter.js. The schedule belongs to the user the app runs for, and to that app. On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.onSchedule(options)
+```
+
+## Parameters
+
+#### `options` (Object) (required)
+
+Give exactly one of `every`, `cron` or `at`.
+
+- `every` (Number | String): Run at this interval, in milliseconds or as a duration such as `'30m'`, `'6h'` or `'1d'`. The units are `s`, `m`, `h`, `d` and `w`. The interval must be a whole number of seconds, at least your plan's [minimum interval](#how-often-a-schedule-can-run), and at most 366 days.
+- `cron` (String): Run on a [cron expression](#cron-expressions), such as `'0 9 * * 1-5'` for 09:00 on weekdays.
+- `tz` (String): The IANA time zone the `cron` expression runs in, such as `'America/New_York'`. Defaults to `'UTC'`. Only allowed with `cron`. See [Time zones and daylight saving time](#time-zones-and-daylight-saving-time).
+- `at` (Number | String | Date): Run once, at this [time](#times). It must be at least 1 second and at most 366 days ahead. A schedule with `at` takes no `startAt`, `expiresAt` or `maxRuns`.
+- `startAt` (Number | String | Date): For `every`, the first run. Without it, the first run is one interval after this call. A `startAt` in the past sets where the intervals line up, and the first run is the next one after this call. For example, `every: '1d'` with `startAt: '2026-01-01'` runs at midnight UTC every day. For `cron`, the earliest time the schedule may run. At most 366 days ahead.
+- `expiresAt` (Number | String | Date): The last time the schedule may run. A run due at exactly this time still runs.
+- `maxRuns` (Number): How many slots the schedule has, from 1 to 100,000. It ends at its `maxRuns`-th slot counted from the first. It counts slots, not runs that happened, so slots skipped while the schedule was behind or suspended (see [When a schedule runs](#when-a-schedule-runs)) still count, and the retries of one slot count once. With `expiresAt` too, the one that comes first ends the schedule.
+- `handlerName` (String) (required): The published handler each run calls.
+- `handler` (Function | String | Object): The handler source you wrote this schedule against, as a function, a source string, or `{ file: '~/AppData/…/handler.js' }`. Only its hash is sent. The schedule is created only if it matches what's published under `handlerName`. It never runs in this client.
+- `handlerHash` (String): The SHA-256 hex digest of the handler source, in place of `handler`.
+- `context` (Object): Values the handler needs, passed to every run as a frozen `ctx`. Up to 4 KB serialized. It's read once, at this call, and can't be changed later. To change it, remove the schedule and create a new one.
+
+### Times
+
+The `at`, `startAt` and `expiresAt` options each take one of these:
+
+- An ISO-8601 string with a zone, such as `'2026-11-01T18:30:00Z'` or `'2026-11-01T18:30+02:00'`. Seconds and up to 3 fraction digits are optional.
+- A date alone, such as `'2026-11-01'`, which is read as midnight UTC.
+- A `Date`.
+- A whole number of milliseconds since the epoch.
+
+Times are rounded up to the next whole second. A string with a time and no zone, such as `'2026-11-01T18:30'`, is refused with `events_schedule_invalid`, because it could be read in any zone.
+
+These times are in milliseconds, unlike `expiresAt` on [`onPersistent()`](/Events/onPersistent/), which takes seconds. Every time a schedule reports, including `event.scheduledFor`, is in milliseconds too.
+
+## Cron expressions
+
+A `cron` expression has the 5 standard fields, separated by spaces:
+
+| Field | Values |
+| --- | --- |
+| Minute | `0`-`59` |
+| Hour | `0`-`23` |
+| Day of the month | `1`-`31` |
+| Month | `1`-`12`, or `JAN`-`DEC` |
+| Day of the week | `0`-`7`, where `0` and `7` are Sunday, or `SUN`-`SAT` |
+
+Each field takes a number, `*` for every value, a list such as `1,15`, a range such as `MON-FRI`, or a step such as `*/15` or `0-30/10`. Names can be in any case.
+
+The expression can also be one of these nicknames:
+
+| Nickname | Runs |
+| --- | --- |
+| `@yearly`, `@annually` | `0 0 1 1 *`, midnight on January 1 |
+| `@monthly` | `0 0 1 * *`, midnight on the first of the month |
+| `@weekly` | `0 0 * * 0`, midnight on Sunday |
+| `@daily`, `@midnight` | `0 0 * * *`, every midnight |
+| `@hourly` | `0 * * * *`, the start of every hour |
+
+Some cron syntax isn't supported, and is refused with `events_schedule_invalid`:
+
+- A seconds or year field, so 6 or 7 fields.
+- `?`, `L`, `W` and `#`, and a day of the week starting with `+`.
+- `@reboot`.
+- An expression over 128 characters.
+
+When neither the day of the month nor the day of the week is `*`, a day matches if either one matches, as in standard cron. So `0 9 13 * 5` runs at 09:00 on the 13th of each month and at 09:00 every Friday. A day that a month doesn't have is skipped, so `0 0 31 * *` runs only in months with 31 days.
+
+The expression has to run within 366 days of the call. An expression that never runs, such as `0 0 31 2 *`, is refused with `events_schedule_never_fires`. So is `0 0 29 2 *` while the next February 29 is more than 366 days away.
+
+### Time zones and daylight saving time
+
+A `cron` expression runs in `tz`. For example, `0 9 * * 1-5` with `tz: 'America/New_York'` runs at 09:00 New York time all year, which is 13:00 UTC in the summer and 14:00 UTC in the winter.
+
+The `tz` option takes an IANA name. A UTC offset such as `'+02:00'` is refused with `events_schedule_invalid`, since an offset has no daylight saving rules. The schedule reports the zone under its canonical name, so `'US/Eastern'` comes back as `'America/New_York'`.
+
+When the clocks change:
+
+- An expression with one minute and one hour, such as `30 2 * * *`, runs once that day. When its time is skipped because the clocks go forward, it runs at the first minute after the gap (03:00 in New York). When its time happens twice because the clocks go back, it runs once.
+- An expression with a list, range, step or `*` in the minute or hour field, such as `0,30 1 * * *` or `0 * * * *`, skips the runs that fall in the gap when the clocks go forward. When the clocks go back, it runs at each repeated time twice. Each of those runs has its own `scheduledFor` and `event.id`.
+
+## How often a schedule can run
+
+A schedule can't run more often than your plan's minimum interval, which is 1 minute on a paid plan and 15 minutes on a free one. For `every`, the interval is what's checked. For `cron`, it's the smallest gap between two runs, checked over the next 10,000 runs or 400 days, whichever ends first. For example, `0-4 9 * * *` runs at 09:00, 09:01, 09:02, 09:03 and 09:04 each day. Those runs are 1 minute apart, so a free plan can't use it. A schedule under the minimum is refused with `events_schedule_too_frequent`.
+
+The plan also limits how many schedules the user can hold for your app and in total, and how many runs per hour all of them can add up to. Over those limits, the call fails with `events_schedule_limit` or `events_schedule_budget`. With `maxRuns` on a `cron` schedule, the last run has to fall inside those same 10,000 runs or 400 days, or the call fails with `events_schedule_invalid`. See [Rate Limits and Quotas](/rate-limits-and-quotas/#schedules) for the numbers.
+
+The plan's limits are checked when a schedule is created. A schedule keeps running if the user later moves to a plan with lower limits. Schedules created after that are held to the new plan.
+
+## When a schedule runs
+
+Each run is due at a time, called its slot, which the handler gets as `event.scheduledFor`. A run usually starts within 60 seconds of its slot.
+
+So that schedules due at the same moment don't all start at once, Puter starts each schedule's runs a fixed delay after their slots. The delay comes from the schedule's id, and is a whole number of seconds under 30 seconds or a tenth of the schedule's smallest gap, whichever is smaller. For an `at` schedule it's under 10 seconds. So a schedule that runs every minute has a delay of at most 5 seconds.
+
+A run can start later than 60 seconds after its slot when it's being retried, when your app's schedules for this user go over 200 runs a minute, while your events worker is being deployed, or while the user's account is suspended. The handler can compare `event.firedAt` with `event.scheduledFor` to see how late a run is.
+
+Puter starts one attempt at a time for each schedule, and the next slot waits until the current run finishes or is given up. A handler that's still running when its 30 seconds are up can overlap its own retry, which has the same `event.id`.
+
+When a schedule falls behind, for example while a run was being retried, it doesn't make up every slot it missed. It runs the latest slot that's due, once, and `event.missed` counts the slots it skipped. A schedule that resumes after a suspension waits for its next slot, and that run's `missed` counts the slots it skipped while it was suspended.
+
+## Delivery guarantees
+
+Runs are at-least-once. A retry, or a handler that outlives its timeout, can run one slot twice. Every attempt at one slot has the same `event.id`, `<scheduleId>:<scheduledFor>`, so record it once the work is done and skip ids you've already handled:
+
+```js
+await puter.events.handlers.publish('sendDigest', async ({ event, ctx, user }) => {
+    const doneKey = `digest:${event.id}`;
+    if (await user.kv.get(doneKey)) return;
+
+    await fetch(ctx.webhookUrl, { method: 'POST', body: `Digest for ${event.scheduledFor}` });
+
+    await user.kv.set(doneKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}, { appUid });
+```
+
+## The event
+
+The handler is called with `event`, `ctx`, `user` and `fetch`, the same bindings a [persistent subscription's](/Events/onPersistent/#where-the-handler-runs) handler gets in the events worker. The `ctx` binding is the frozen `context`. There's no `ack`.
+
+A run's `event` carries:
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `subject` | String | `schedule:<scheduleId>`. |
+| `op` | String | Always `'tick'`. |
+| `id` | String | `<scheduleId>:<scheduledFor>`. The same on every attempt at one run, so use it to skip work you've already done. |
+| `scheduleId` | String | The schedule this run belongs to. |
+| `scheduledFor` | Number | The slot this run is for, in milliseconds since the epoch. |
+| `firedAt` | Number | When this attempt was sent to the handler, in milliseconds since the epoch. |
+| `missed` | Number | Slots skipped since the last run, for example while the schedule was suspended. Only this run happens for them. |
+| `missedCapped` | Boolean | Present, and `true`, when `missed` stopped counting at 1,000. |
+| `attempt` | Number | Attempts at this run, this one included. `1` on the first try. |
+
+The `user` binding is valid for 15 minutes, so use it only during the run. Writes made through `user` count toward the same [handler chain](/Events/onPersistent/#handlers-that-trigger-handlers) limit as a subscription's, starting from the depth of the call that created the schedule.
+
+## What happens when a handler fails
+
+| The handler | Result |
+| --- | --- |
+| Resolves | Done. The schedule moves on to its next slot. |
+| Throws | The run is dropped and the schedule moves on to its next slot. It counts as a failure. |
+| Throws a temporary error, meaning a network error, a timeout, a rate limit (`429`), a `5xx`, or any error with `retry: true` | Retried after 2 seconds, then 4, 8 and so on, up to 5 minutes apart, with the same `event.id`. It counts as a failure. |
+| Runs past 30 seconds | Retried on the same schedule. It counts as a failure. |
+
+A run that fails because of Puter doesn't count as a failure. It's retried on the same schedule, and waits longer if Puter's answer asked for a longer wait, up to 5 minutes. To drop the run in that case anyway, throw an error with `terminal: true`. The rules are the same as for a [`queue` subscription](/Events/onPersistent/#what-happens-when-a-handler-fails), except that the retries stop once the schedule's next run is due, or after 24 hours for an `at` schedule or a last run. The run is then dropped without billing.
+
+Five failures in a row suspend the schedule with `failures`. The developer is notified, and publishing the handler again resumes it.
+
+## Background runs need consent
+
+A schedule runs your handler while the user isn't there, so it needs the per-app permission `events:background`, as [`onPersistent()`](/Events/onPersistent/#background-delivery-needs-consent) does. Without it, the call fails with `events_background_consent_required`. To ask for it, call [`puter.perms.request()`](/Perms/request/) with the `'backgroundEvents'` resource:
+
+```js
+await puter.perms.request('backgroundEvents');
+```
+
+When the user revokes it, every schedule your app holds for them ends with `permission_revoked`. Granting it again doesn't bring them back, so create them again.
+
+## Suspended and ended schedules
+
+A schedule can stop running without being removed. It's then suspended, and [`listSchedules()`](/Events/listSchedules/) shows `state: 'suspended'` and a `suspendedReason`:
+
+| `suspendedReason` | Cause | Resumes when |
+| --- | --- | --- |
+| `handler_not_found` | Its handler was removed, or the app's [events worker](/Events/workers/) was destroyed. | A handler is published under that name again. |
+| `failures` | Its handler failed 5 times in a row. | The handler is published again. |
+| `no_credit` | The user ran out of credit. | The balance is topped up (checked every few minutes). |
+
+A suspended schedule isn't billed. One that stays suspended for 30 days ends.
+
+An ended schedule never runs again. It has `state: 'ended'` and an `endReason`, and stays in [`listSchedules()`](/Events/listSchedules/) for 7 days, after which it is deleted:
+
+| `endReason` | Cause |
+| --- | --- |
+| `removed` | It was removed with [`removeSchedule()`](/Events/removeSchedule/) or `off()`. |
+| `ended` | Its last run is done (an `at` schedule, `expiresAt` or `maxRuns`), or it stayed suspended for 30 days. |
+| `permission_revoked` | The user revoked `events:background` for your app. |
+| `uninstalled` | The user uninstalled your app. |
+| `app_deleted` | The app was deleted. |
+| `holder_gone`, `app_gone` | The user's account or the app no longer exists. |
+| `orphaned` | A create or remove that failed partway left it incomplete, and Puter cleaned it up. |
+| `delivered`, `dropped` | A notification sent later with [`notify()`](/Notifications/notify/#send-it-later) was sent, or couldn't be. |
+
+When schedules end with `permission_revoked`, the user's app gets an `app.events.ended` notification about them, which you can read with [`puter.events.fetch()`](/Events/fetch/).
+
+## Return value
+
+A `Promise` that resolves to the schedule. All times are in milliseconds since the epoch.
+
+- `scheduleId` (String): Its id, which [`getSchedule()`](/Events/getSchedule/) and [`removeSchedule()`](/Events/removeSchedule/) take.
+- `kind` (String): `'recurring'` for `every` and `cron`, `'once'` for `at`, and `'deferred'` for a notification sent later with [`notify()`](/Notifications/notify/#send-it-later).
+- `appUid` (String): The app that created it.
+- `handlerName` (String | null): The handler each run calls, or `null` once the schedule has ended.
+- `contextKeys` (Array | null), `contextHash` (String | null): The key names and a hash of the stored context, never its values. The `contextKeys` field is `null` when there's no context, and once the schedule has ended.
+- `every` (Number | null): The interval, in milliseconds, of an `every` schedule.
+- `cron` (String | null): The expression of a `cron` schedule.
+- `tz` (String): The time zone of a `cron` schedule, and `'UTC'` for the others.
+- `startAt` (Number): The first time it may run. For `at`, its only run.
+- `endsAt` (Number | null): The last time it may run, from `expiresAt` or `maxRuns`. For `at`, its only run.
+- `state` (String): `'active'`, `'suspended'`, `'sending'`, or `'ended'`. A delayed notification is `'sending'` once it has started to go out.
+- `suspendedReason` (String | null): Why it's [suspended](#suspended-and-ended-schedules).
+- `endReason` (String | null), `endedAt` (Number | null): Why and when it [ended](#suspended-and-ended-schedules).
+- `nextRunAt` (Number | null): The next slot of the schedule, worked out from its definition and the clock. A late or retried run for an earlier slot can still happen first. After the last slot is due, it stays at that slot until the run for it is done. It's `null` while the schedule is suspended or sending, and once it has ended.
+- `createdAt` (Number): When it was created.
+- `off()` (Function): Removes the schedule. Same as [`puter.events.removeSchedule(scheduleId)`](/Events/removeSchedule/). Only on the object `onSchedule()` returns.
+
+The promise rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `invalid_request` | `options` isn't an object. |
+| `events_handler_required` | There's no `handlerName`. |
+| `events_handler_name_required` | `handler` was given without `handlerName`. |
+| `events_handler_free_variable` | The handler uses an outside variable. The message names it. |
+| `events_handler_invalid` | `handler` isn't a function, a source string, or `{ file }`. |
+| `events_handler_hash_unavailable` | This environment has no `crypto.subtle`, so `handler` can't be hashed. Pass `handlerName` alone. |
+| `events_context_invalid` | `context` isn't JSON-serializable. |
+| `events_context_too_large` | The serialized `context` is over 4 KB. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `events_schedule_app_required` | The call wasn't made by an app, for example from Node.js with an account token. |
+| `events_durable_requires_account` | The call came from a scoped access token, such as the one in a [`getReadURL()`](/FS/getReadURL/) URL, or from a temporary (guest) account. |
+| `too_many_requests` | Over the create rate limit, which is shared with subscribing. |
+| `events_background_consent_required` | The user hasn't granted your app `events:background`. |
+| `events_schedule_invalid` | An option is malformed. For example, more than one of `every`, `cron` and `at`, an `every` that isn't a whole number of seconds, a time with no zone, an `at` less than 1 second or more than 366 days ahead, an unsupported cron expression, an unknown `tz`, or `tz` without `cron`. The message says which. |
+| `events_schedule_never_fires` | The schedule would never run. Its first run or its `startAt` is more than 366 days away, or it ends before its first run. |
+| `events_schedule_too_frequent` | It would run more often than your plan's [minimum interval](#how-often-a-schedule-can-run). |
+| `events_schedule_too_deep` | The call was made from a handler at the end of a [handler chain](/Events/onPersistent/#handlers-that-trigger-handlers), so the schedule could never run. |
+| `bad_request` | `handlerName` is empty or over 128 characters, or `handlerHash` isn't a SHA-256 hex digest. |
+| `events_handler_not_found` | Nothing is published under `handlerName`. The schedule isn't created. |
+| `events_handler_hash_mismatch` | The published handler isn't the source you passed as `handler`. |
+| `events_schedule_limit` | The user holds as many schedules as their plan allows, for your app or in total. |
+| `events_schedule_budget` | The schedule would make your app's schedules, or all of the user's schedules, run more times an hour than their plan allows. |
+| `events_schedule_busy` | Puter is checking too many cron expressions at once. Try again in a moment. |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+The handler is published once, by the app's developer, from a deploy script or the browser console. The schedule is created from the app, for each user.
+
+<strong class="example-title">Run a handler on weekday mornings in the user's time zone</strong>
+
+```js
+// Deploy script, run as the app's developer
+const app = await puter.apps.get('daily-digest');
+
+await puter.events.handlers.publish('sendDigest', async ({ event, ctx, user }) => {
+    const doneKey = `digest:${event.id}`;
+    if (await user.kv.get(doneKey)) return;
+
+    const notes = await user.kv.list('note:*', true);
+    await user.notifications.notify({
+        title: `Good morning, ${ctx.name}`,
+        text: `You have ${notes.length} notes.`,
+    });
+
+    await user.kv.set(doneKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}, { appUid: app.uid });
+```
+
+```js
+// In the app, signed in as the user
+await puter.perms.request('backgroundEvents');
+
+const schedule = await puter.events.onSchedule({
+    cron: '0 8 * * 1-5',
+    tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
+    handlerName: 'sendDigest',
+    context: { name: 'Ana' },
+});
+if (schedule.nextRunAt !== null) console.log(`next digest at ${new Date(schedule.nextRunAt).toString()}`);
+```
+
+<strong class="example-title">Run once, an hour from now</strong>
+
+```js
+const reminder = await puter.events.onSchedule({
+    at: Date.now() + 60 * 60 * 1000,
+    handlerName: 'remind',
+    context: { todoId: 'todo-42' },
+});
+
+// Changed your mind
+await reminder.off();
+```
+
+<strong class="example-title">Run every 6 hours, on the hour, 28 times</strong>
+
+```js
+await puter.events.onSchedule({
+    every: '6h',
+    startAt: '2026-01-01T00:00:00Z',
+    maxRuns: 28,
+    handlerName: 'syncFeeds',
+});
+```

src/docs/src/Events/removeSchedule.md

@@ -0,0 +1,72 @@
+---
+title: puter.events.removeSchedule()
+description: End a schedule, or cancel a notification stored to send later.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+Ends a schedule created with [`puter.events.onSchedule()`](/Events/onSchedule/). It doesn't run again, and it stays in [`listSchedules()`](/Events/listSchedules/) for 7 days with `endReason: 'removed'`. A run that already started isn't stopped.
+
+It also cancels a notification your app stored to send later with [`puter.notifications.notify()`](/Notifications/notify/#send-it-later), the same as [`puter.notifications.cancel()`](/Notifications/cancel/).
+
+The `off()` method on the schedule [`onSchedule()`](/Events/onSchedule/) returns does the same thing.
+
+On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.removeSchedule(scheduleId)
+```
+
+## Parameters
+
+#### `scheduleId` (String) (required)
+The `scheduleId` from [`onSchedule()`](/Events/onSchedule/), [`notify()`](/Notifications/notify/#send-it-later) or [`listSchedules()`](/Events/listSchedules/).
+
+## Return value
+
+A `Promise` that resolves when the schedule has ended.
+
+An app can only remove schedules it created. An account session can remove any of the user's schedules, including ones left by apps they no longer use. A scoped access token can't remove any.
+
+An id this caller doesn't hold, or a schedule that already ended, is reported as not existing, so removing a schedule twice rejects the second time. With `events_schedule_sending`, `auth_canceled`, `too_many_requests` or `events_disabled`, nothing was removed. When the call fails any other way, such as a network error or `events_failed`, the removal may still have happened, so read the schedule again with [`getSchedule()`](/Events/getSchedule/) to find out.
+
+It rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `events_schedule_not_found` | No such schedule, it isn't this caller's, or it already ended. Also for an empty `scheduleId`. |
+| `events_schedule_sending` | A notification stored to send later has started to go out, so it can't be cancelled. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `too_many_requests` | Over the remove rate limit, which is shared with [`puter.events.unsubscribe()`](/Events/unsubscribe/). |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">Cancel a notification, then read it</strong>
+
+```html;events-remove-schedule
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            const later = await puter.notifications.notify({
+                title: 'Your trial ends tomorrow',
+                sendAt: Date.now() + 24 * 60 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            await puter.events.removeSchedule(later.scheduleId);
+            puter.print('cancelled<br>');
+
+            // The ended schedule can still be read for 7 days
+            const ended = await puter.events.getSchedule(later.scheduleId);
+            puter.print(`${ended.state} (${ended.endReason})<br>`);
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Events/subjects.md

@@ -176,6 +176,8 @@ A key-value change carries `key` instead of `uid` and `path`, and different ops:
 
 A notification carries the fields listed on [`fetch()`](/Events/fetch/#return-value).
 
+A [schedule](/Events/onSchedule/) has no subject you can subscribe to. Its runs carry `op: 'tick'` and the fields listed on [`onSchedule()`](/Events/onSchedule/#the-event).
+
 Events never say who made a change. On a shared folder, that would tell every subscriber who else has access.
 
 [`puter.kv.flush()`](/KV/flush/) delivers nothing, since the keys a flush can list aren't reliably the keys it removed.

src/docs/src/Events/unsubscribe.md

@@ -33,6 +33,7 @@ The promise rejects with `{ message, code }`:
 | `code` | Meaning |
 | --- | --- |
 | `subscription_does_not_exist` | No such subscription, or it isn't yours to end. |
+| `events_schedule_not_found` | The subscription belongs to a [schedule](/Events/onSchedule/). Remove it with [`removeSchedule()`](/Events/removeSchedule/) instead. |
 | `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
 | `too_many_requests` | Over the unsubscribe rate limit, which is separate from the subscribe one. |
 | `events_disabled` | Events aren't enabled on this server. |

src/docs/src/Events/update.md

@@ -36,6 +36,7 @@ The promise rejects with `{ message, code }`:
 | `code` | Meaning |
 | --- | --- |
 | `subscription_does_not_exist` | No such subscription, or it isn't yours to change. |
+| `events_schedule_immutable` | The subscription belongs to a [schedule](/Events/onSchedule/), and a schedule's `context` can't be changed. Remove the schedule and create a new one. |
 | `invalid_request` | `context` is missing. |
 | `events_context_too_large` | `context` is over 4 KB serialized. |
 | `events_context_invalid` | `context` can't be serialized as JSON. |

src/docs/src/Events/workers.md

@@ -6,7 +6,7 @@ platforms: [websites, apps, nodejs, workers]
 
 <div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
 
-An **events worker** runs an app's published [handlers](/Events/handlers/). It runs them for every event on a subscription with a `handlerName`, whether or not a client is connected, and for a [`queue`](/Events/onPersistent/#choosing-a-mode) delivery once no connected client takes it. Each app has at most one, created when it publishes its first handler, however many handlers it publishes.
+An **events worker** runs an app's published [handlers](/Events/handlers/). It runs them for every event on a subscription with a `handlerName`, whether or not a client is connected, for a [`queue`](/Events/onPersistent/#choosing-a-mode) delivery once no connected client takes it, and for every run of a [schedule](/Events/onSchedule/). Each app has at most one, created when it publishes its first handler, however many handlers it publishes.
 
 A hosted Puter server may bill each app's events worker as a monthly cost, even if nothing ever delivers to it. Use this API to see which apps have one and remove the ones you don't need.
 
@@ -43,11 +43,11 @@ Resolves to `{ items, cursor, deployable }`:
 puter.events.workers.destroy(appUid)
 ```
 
-Removes every handler the app has published, the same as calling [`puter.events.handlers.remove()`](/Events/handlers/) on each. Their subscriptions are suspended with `handler_not_found`, not deleted, and publishing the handlers again resumes them.
+Removes every handler the app has published, the same as calling [`puter.events.handlers.remove()`](/Events/handlers/) on each. Their subscriptions and schedules are suspended with `handler_not_found`, not deleted, and publishing the handlers again resumes them.
 
 It also ends the worker session background deliveries ran under, for every user. The next delivery after a new publish starts a new one.
 
-Resolves to `{ appUid, removed, suspended }`, the number of handlers deleted and the number of subscriptions suspended. An app with nothing published rejects with `events_handler_not_found`.
+Resolves to `{ appUid, removed, suspended }`, the number of handlers deleted and the number of subscriptions and schedules suspended. An app with nothing published rejects with `events_handler_not_found`.
 
 ## Errors
 

src/docs/src/Notifications.md

@@ -29,6 +29,8 @@ A notification has a `title`, and optionally `text` and an `icon`. A `target` pa
 
 [`notify()`](/Notifications/notify/) lists every field. [Rate Limits and Quotas](/rate-limits-and-quotas/#notifications) lists the size limits and how many notifications and pushes your app can send each user.
 
+To send a notification at a set time instead of now, pass `sendAt` to [`notify()`](/Notifications/notify/#send-it-later). It goes out then, even when your app is closed.
+
 ## Who you can notify
 
 A notification goes to the user your app runs for. To notify another user, such as a chat member, share data with them in a [key-value share](/KV/share/) made with `notify: true`, and send through the share's handle. Each side of the share can notify the other.
@@ -134,6 +136,7 @@ Your app's notifications are events on the `notif:app-user` [subject](/Events/su
 ## Functions
 
 - **[`puter.notifications.notify()`](/Notifications/notify/)** - Send a notification to the user, or to the other side of a key-value share
+- **[`puter.notifications.cancel()`](/Notifications/cancel/)** - Cancel a notification stored to send later
 - **[`puter.notifications.onNotification()`](/Notifications/onNotification/)** - Receive your app's notifications, including the ones sent while it was closed
 - **[`puter.notifications.onPush()`](/Notifications/onPush/)** - Say what a push shows for one channel, and handle its notifications while the page is open
 - **[`puter.notifications.on('click')`](/Notifications/on/)** - React when the user clicks one of your notifications
@@ -152,6 +155,7 @@ For Puter's own apps, which manage where pushes go:
 ## Examples
 
 - [Send a Notification](/playground/notifications-notify/)
+- [Send a Notification Later](/playground/notifications-send-later/)
 - [Push notifications for a website](/Notifications/push/#a-minimal-page)
 
 ## Recipes

src/docs/src/Notifications/cancel.md

@@ -0,0 +1,64 @@
+---
+title: puter.notifications.cancel()
+description: Cancel a notification your app stored to send later, before it goes out.
+platforms: [websites, apps]
+---
+
+Cancels a notification your app stored to send later with [`notify()`](/Notifications/notify/#send-it-later) and `sendAt`. It isn't sent, and it stops counting toward the [limit on what your app has waiting](/rate-limits-and-quotas/#schedules) for the user.
+
+It does the same thing as [`puter.events.removeSchedule()`](/Events/removeSchedule/), and shares its rate limit with it. The cancelled notification stays in [`puter.events.listSchedules()`](/Events/listSchedules/) for 7 days, with `state: 'ended'` and `endReason: 'removed'`.
+
+A notification that has started to go out can't be cancelled. Once it's sent, it's in the user's notifications like any other.
+
+## Syntax
+```js
+puter.notifications.cancel(scheduleId)
+```
+
+## Parameters
+
+#### `scheduleId` (String) (required)
+The `scheduleId` that [`notify()`](/Notifications/notify/#send-it-later) resolved with.
+
+## Return value
+
+A `Promise` that resolves when the notification is cancelled.
+
+It rejects with an error carrying `message` and `code`:
+
+| `code` | Meaning |
+| --- | --- |
+| `events_schedule_not_found` | No such notification, it isn't your app's, or it was already sent or cancelled. Also for an empty `scheduleId`. |
+| `events_schedule_sending` | It has started to go out, so it can't be cancelled. |
+| `too_many_requests` | Over the cancel rate limit, which is shared with [`puter.events.removeSchedule()`](/Events/removeSchedule/) and [`puter.events.unsubscribe()`](/Events/unsubscribe/). |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `events_disabled` | Sending later isn't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">Store a reminder, then cancel it</strong>
+
+```html;notifications-send-later
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            // (1) Send it in 30 minutes instead of now
+            const later = await puter.notifications.notify({
+                title: 'Your timer is done',
+                target: '/timers',
+                sendAt: Date.now() + 30 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+            puter.print(`stored ${later.scheduleId}, goes out ${new Date(later.sendAt).toLocaleTimeString()}<br>`);
+
+            // (2) The user stopped the timer, so it shouldn't go out
+            await puter.notifications.cancel(later.scheduleId);
+            puter.print('cancelled<br>');
+        })();
+    </script>
+</body>
+</html>
+```