Merge remote-tracking branch 'origin/main' into jas-batch

Invocations that come due within a few milliseconds go out as one
/invoke-batch request (up to events.invokeBatchSize, default 60). The
dispatcher runs a few at a time and streams a started line and an answer line
per call, so a slow handler delays no other call's answer. A call's timeout
runs from its started line, so time queued behind other calls is never the
handler's; queue time comes out of the connection-wait share instead, and a
call still queued past it is never started and fails as Puter's.

A lone call still uses /invoke, and a dispatcher without the batch route gets
single calls for five minutes before the batch route is tried again.

src/backend/clients/database/DatabaseClient.test.ts

@@ -285,6 +285,18 @@ describe('AbstractDatabaseClient — SQL generation', () => {
         );
     });
 
+    it('reads the database clock as epoch milliseconds in each dialect', () => {
+        expect(sqlite().nowMsExpr()).toBe(
+            "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+        );
+        expect(mysql().nowMsExpr()).toBe(
+            "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        );
+        expect(postgres().nowMsExpr()).toBe(
+            'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+        );
+    });
+
     it('describes a stored timestamp the way each driver reads it back', () => {
         const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
         expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');

src/backend/clients/database/DatabaseClient.ts

@@ -401,6 +401,21 @@ export class AbstractDatabaseClient extends PuterClient {
         });
     }
 
+    /**
+     * The database's own clock as epoch milliseconds, so nodes whose clocks
+     * disagree still agree on it. The mysql form ignores the session time zone,
+     * so a DST change can't shift it.
+     */
+    nowMsExpr(): string {
+        return this.case({
+            sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
+            postgres:
+                'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
+            otherwise:
+                "(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
+        });
+    }
+
     /**
      * What a read of a timestamp column holding `seconds` returns: sqlite gives
      * back the UTC text it stored, mysql2 and pg a `Date` read in node's time

src/backend/clients/database/SqliteDatabaseClient.test.ts

@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
 import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
 
 /** Highest schema version the migration table can reach. */
-const CURRENT_SCHEMA_VERSION = 95;
+const CURRENT_SCHEMA_VERSION = 96;
 
 /**
  * These suites migrate real files on disk. Idle they finish in well under a
@@ -724,6 +724,17 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         ]);
     });
 
+    it('keeps notification.send_key unique', async () => {
+        const rows = await client.read('PRAGMA index_list(`notification`)');
+        expect(
+            rows.find((r) => r.name === 'idx_notification_send_key'),
+        ).toMatchObject({ unique: 1 });
+        const columns = await client.read(
+            'PRAGMA index_info(`idx_notification_send_key`)',
+        );
+        expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
+    });
+
     it('leaves an already-migrated database untouched on a second boot', async () => {
         const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
         const path = join(dir, 'nested', 'puter.sqlite');

src/backend/clients/database/SqliteDatabaseClient.ts

@@ -134,6 +134,7 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
     [92, ['0097_user-to-user-holder-permission-index.sql']],
     [93, ['0098_notification-groups.sql']],
     [94, ['0099_kv-share-handle-notify.sql']],
+    [95, ['0100_notification-send-key.sql']],
 ];
 
 export class SqliteDatabaseClient extends AbstractDatabaseClient {

src/backend/clients/database/migrations/mysql/mysql_mig_54.sql

@@ -0,0 +1,43 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
+-- rationale. The key is a uuid, so ascii and compared byte for byte. No
+-- per-file applied-state tracking, so every step is guarded and a replay does
+-- nothing.
+
+CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_send_key_index()
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_send_key'
+  ) THEN
+    ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
+      (`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
+  END IF;
+END//
+DELIMITER ;
+
+CALL _puter_add_notification_send_key_index();
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;

src/backend/clients/database/migrations/postgres/postgres_mig_43.sql

@@ -0,0 +1,24 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
+-- rationale. Idempotent via IF NOT EXISTS.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
+    ON notification (send_key);

src/backend/clients/database/migrations/sqlite/0100_notification-send-key.sql

@@ -0,0 +1,27 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- `notification`.`send_key`: the key of the delayed send (a `deferred`
+-- schedule) that wrote this row or last replaced it, derived from the
+-- schedule and its slot. Unique, so one slot can never write two rows, and a
+-- send that crashed after writing finds its row again by it. Null on every row
+-- a delayed send never touched.
+
+ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
+    ON `notification` (`send_key`);

src/backend/clients/event/types.ts

@@ -581,6 +581,8 @@ export type EventMap = {
         updated?: true;
         /** Names this one send, for a receipt asked about it. */
         sendId: string;
+        /** The same `sendId` may be emitted again, as a delayed send is. */
+        replayable?: true;
     };
     /**
      * One notification, addressed at whatever sockets each region holds for the

src/backend/clients/events/EventsWorkerInvokerClient.test.ts

@@ -28,13 +28,16 @@ import type { IConfig } from '../../types.js';
 import {
     DispatcherInvokeTransport,
     EVENTS_DEPLOYED_HEADER,
+    EVENTS_DISPATCH_BATCH_PATH,
     EVENTS_DISPATCH_ERROR_HEADER,
     EVENTS_DISPATCH_PATH,
     EVENTS_ERROR_HEADER,
     EVENTS_HANDLED_HEADER,
     EventsWorkerInvokerClient,
     INVOKE_CONNECTIONS,
+    SCHEDULE_INVOKE_CONNECTIONS,
     outcomeForStatus,
+    parseAnswerLine,
     type WorkerInvokeRequest,
 } from './EventsWorkerInvokerClient.js';
 
@@ -73,10 +76,7 @@ const refusedFetch: FetchImpl = (async () => {
 }) as unknown as FetchImpl;
 
 /** A stub fetch that records the URL it was called with. */
-const capturingFetch = (
-    calls: string[],
-    status = 200,
-): FetchImpl =>
+const capturingFetch = (calls: string[], status = 200): FetchImpl =>
     (async (url: string) => {
         calls.push(url);
         return new Response(null, { status });
@@ -84,9 +84,13 @@ const capturingFetch = (
 
 describe('DispatcherInvokeTransport', () => {
     it('reports a handled 400 with no error — the handler`s own refusal', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 400,
             handled: true,
@@ -94,9 +98,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('reports an unmarked 404 with no dispatch-error header as unhandled', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(404),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(404),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 404,
             handled: false,
@@ -104,9 +112,15 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('turns a dispatch-error header into a null status with the reason', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(502, { [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(502, {
+                    [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed',
+                }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: null,
             error: 'dispatcher: deploy-failed (502)',
@@ -120,9 +134,13 @@ describe('DispatcherInvokeTransport', () => {
             headers.push(init?.headers as Record<string, string>);
             return new Response(null, { status: 200 });
         }) as unknown as FetchImpl;
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl,
+            },
+        );
 
         await transport.send(CALL);
         await transport.send({ ...CALL, deployed: true });
@@ -132,9 +150,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('reports a handled 200 as settled with no error', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 200,
             handled: true,
@@ -142,9 +164,13 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('passes a 429 through as-is', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(429),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(429),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 429,
             handled: false,
@@ -152,18 +178,26 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('answers null and says so when the request times out', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: hangingFetch,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: hangingFetch,
+            },
+        );
         const result = await transport.send({ ...CALL, timeoutMs: 10 });
         expect(result.status).toBeNull();
         expect(result.timedOut).toBe(true);
     });
 
     it('answers null without a timeout when the request fails outright', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: refusedFetch,
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: refusedFetch,
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: null,
             error: 'fetch failed',
@@ -187,12 +221,16 @@ describe('DispatcherInvokeTransport', () => {
     });
 
     it('surfaces the runtime`s own error header alongside its status', async () => {
-        const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
-            fetchImpl: stubFetch(500, {
-                [EVENTS_HANDLED_HEADER]: '1',
-                [EVENTS_ERROR_HEADER]: 'handler-threw',
-            }),
-        });
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            {
+                fetchImpl: stubFetch(500, {
+                    [EVENTS_HANDLED_HEADER]: '1',
+                    [EVENTS_ERROR_HEADER]: 'handler-threw',
+                }),
+            },
+        );
         expect(await transport.send(CALL)).toEqual({
             status: 500,
             handled: true,
@@ -208,7 +246,9 @@ describe('DispatcherInvokeTransport', () => {
             { fetchImpl: capturingFetch(calls) },
         );
         await transport.send(CALL);
-        expect(calls).toEqual([`http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`]);
+        expect(calls).toEqual([
+            `http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`,
+        ]);
     });
 });
 
@@ -237,10 +277,12 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
     /** Every connection taken by a call nobody has answered yet. */
     const saturated = async () => {
         const { fetchImpl, held } = heldFetch();
+        // Slots are per call, batched or not; one request per call keeps
+        // the count readable.
         const transport = new DispatcherInvokeTransport(
             'http://dispatcher',
             's',
-            { fetchImpl },
+            { fetchImpl, batchSize: 1 },
         );
         const busy = Array.from({ length: INVOKE_CONNECTIONS }, () =>
             transport.send(CALL),
@@ -303,6 +345,534 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
     });
 });
 
+describe('DispatcherInvokeTransport schedule lane', () => {
+    const heldFetch = () => {
+        const held: Array<{ answer: () => void; body: string }> = [];
+        const fetchImpl = ((_url: string, init: RequestInit) =>
+            new Promise((resolve) => {
+                held.push({
+                    body: String(init.body),
+                    answer: () =>
+                        resolve(
+                            new Response(null, {
+                                status: 200,
+                                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                            }),
+                        ),
+                });
+            })) as unknown as FetchImpl;
+        return { fetchImpl, held };
+    };
+
+    it('holds schedule runs to their share and keeps serving deliveries', async () => {
+        const { fetchImpl, held } = heldFetch();
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl, batchSize: 1 },
+        );
+        const scheduled = Array.from(
+            { length: SCHEDULE_INVOKE_CONNECTIONS + 5 },
+            () => transport.send({ ...CALL, lane: 'schedule' as const }),
+        );
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
+        );
+        // A delivery waiting behind the schedule queue still gets a connection.
+        const delivery = transport.send(CALL);
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
+        );
+        held[SCHEDULE_INVOKE_CONNECTIONS].answer();
+        await expect(delivery).resolves.toMatchObject({ status: 200 });
+
+        // A schedule run's slot goes to the next schedule run waiting.
+        held[0].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 2),
+        );
+        for (let i = 0; i < 4; i++) held[i + 1].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 6),
+        );
+        for (const call of held) call.answer();
+        await Promise.all(scheduled);
+    });
+
+    it('builds the body after the connection wait, so a stamp is the send time', async () => {
+        const { fetchImpl, held } = heldFetch();
+        const transport = new DispatcherInvokeTransport(
+            'http://dispatcher',
+            's',
+            { fetchImpl, batchSize: 1 },
+        );
+        const busy = Array.from({ length: SCHEDULE_INVOKE_CONNECTIONS }, () =>
+            transport.send({ ...CALL, lane: 'schedule' as const }),
+        );
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
+        );
+        let renders = 0;
+        const stamped = transport.send({
+            ...CALL,
+            lane: 'schedule',
+            render: () => `{"n":${++renders}}`,
+        });
+        held[0].answer();
+        await vi.waitFor(() =>
+            expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
+        );
+        expect(held[SCHEDULE_INVOKE_CONNECTIONS].body).toBe('{"n":1}');
+        for (const call of held) call.answer();
+        await Promise.all([...busy, stamped]);
+    });
+});
+
+describe('DispatcherInvokeTransport batches', () => {
+    interface BatchRequest {
+        url: string;
+        body: { calls: Array<Record<string, unknown>> };
+        signal: AbortSignal;
+        /** Streams one answer line back. */
+        answer: (line: Record<string, unknown>) => void;
+        end: () => void;
+    }
+
+    /**
+     * A fetch whose batch answers stream only when told to; single calls get a
+     * handled 200 straight away.
+     */
+    const streamingFetch = () => {
+        const batches: BatchRequest[] = [];
+        const singles: Array<{ url: string; headers: Record<string, string> }> =
+            [];
+        const fetchImpl = (async (url: string, init: RequestInit) => {
+            if (!url.endsWith(EVENTS_DISPATCH_BATCH_PATH)) {
+                singles.push({
+                    url,
+                    headers: init.headers as Record<string, string>,
+                });
+                return new Response(null, {
+                    status: 200,
+                    headers: { [EVENTS_HANDLED_HEADER]: '1' },
+                });
+            }
+            let controller!: ReadableStreamDefaultController<Uint8Array>;
+            const stream = new ReadableStream<Uint8Array>({
+                start: (c) => {
+                    controller = c;
+                },
+            });
+            const encoder = new TextEncoder();
+            init.signal?.addEventListener('abort', () => {
+                try {
+                    controller.error(init.signal!.reason);
+                } catch {
+                    // already closed
+                }
+            });
+            batches.push({
+                url,
+                body: JSON.parse(String(init.body)),
+                signal: init.signal!,
+                answer: (line) =>
+                    controller.enqueue(
+                        encoder.encode(`${JSON.stringify(line)}\n`),
+                    ),
+                end: () => controller.close(),
+            });
+            return new Response(stream, {
+                status: 200,
+                headers: { 'content-type': 'application/x-ndjson' },
+            });
+        }) as unknown as FetchImpl;
+        return { fetchImpl, batches, singles };
+    };
+
+    const transportFor = (
+        fetchImpl: FetchImpl,
+        options: { batchSize?: number; batchWindowMs?: number } = {},
+    ) =>
+        new DispatcherInvokeTransport('http://dispatcher', 's', {
+            fetchImpl,
+            ...options,
+        });
+
+    it('sends calls made together as one request, and answers each from its own line', async () => {
+        const { fetchImpl, batches, singles } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const first = transport.send({ ...CALL, script: 'evw-a', key: 'k1:a' });
+        const second = transport.send({
+            ...CALL,
+            script: 'evw-b',
+            deployed: true,
+            body: '{"b":1}',
+        });
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+
+        expect(batches[0].url).toBe(
+            `http://dispatcher${EVENTS_DISPATCH_BATCH_PATH}`,
+        );
+        expect(batches[0].body.calls).toEqual([
+            {
+                script: 'evw-a',
+                appUid: 'app-test',
+                key: 'k1:a',
+                body: '{}',
+                waitMs: expect.any(Number),
+            },
+            {
+                script: 'evw-b',
+                appUid: 'app-test',
+                key: 'k1:test',
+                deployed: true,
+                body: '{"b":1}',
+                waitMs: expect.any(Number),
+            },
+        ]);
+        // What is left of half the timeout, the share a call may wait.
+        expect(batches[0].body.calls[0].waitMs).toBeLessThanOrEqual(2_500);
+        expect(batches[0].body.calls[0].waitMs).toBeGreaterThan(2_400);
+
+        // Out of order: each line names the call it answers.
+        batches[0].answer({ i: 1, status: 400, handled: true });
+        await expect(second).resolves.toEqual({ status: 400, handled: true });
+        batches[0].answer({
+            i: 0,
+            status: 503,
+            handled: true,
+            error: 'platform-error',
+            retryAfter: '3',
+        });
+        await expect(first).resolves.toEqual({
+            status: 503,
+            handled: true,
+            error: 'platform-error',
+            retryAfterMs: 3000,
+        });
+        // Every call answered, so the request is let go.
+        expect(batches[0].signal.aborted).toBe(true);
+        expect(singles).toEqual([]);
+    });
+
+    it('sends a call with nobody to batch with on the single route', async () => {
+        const { fetchImpl, batches, singles } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        await expect(transport.send(CALL)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(batches).toEqual([]);
+        expect(singles[0].url).toBe(`http://dispatcher${EVENTS_DISPATCH_PATH}`);
+    });
+
+    it('sends a full batch at once, without waiting out the window', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl, {
+            batchSize: 2,
+            batchWindowMs: 60_000,
+        });
+        const sent = [transport.send(CALL), transport.send(CALL)];
+        const third = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        expect(batches[0].body.calls).toHaveLength(2);
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await Promise.all(sent);
+        transport.close();
+        await expect(third).resolves.toMatchObject({ status: 200 });
+    });
+
+    it('gives each call its own timeout, without failing the rest of the batch', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const quick = transport.send({ ...CALL, timeoutMs: 50 });
+        const slow = transport.send({ ...CALL, timeoutMs: 5_000 });
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, started: true });
+        batches[0].answer({ i: 1, started: true });
+
+        await expect(quick).resolves.toMatchObject({
+            status: null,
+            timedOut: true,
+        });
+        expect(batches[0].signal.aborted).toBe(false);
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await expect(slow).resolves.toEqual({ status: 200, handled: true });
+    });
+
+    it('runs a call`s timeout from its started line, not from the send', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        let settled = false;
+        const queued = transport
+            .send({ ...CALL, timeoutMs: 200 })
+            .then((answer) => {
+                settled = true;
+                return answer;
+            });
+        const other = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+
+        // Queued behind other calls for longer than its whole timeout.
+        await new Promise((resolve) => setTimeout(resolve, 250));
+        expect(settled).toBe(false);
+        batches[0].answer({ i: 0, started: true });
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        expect(settled).toBe(false);
+        await expect(queued).resolves.toMatchObject({
+            status: null,
+            timedOut: true,
+        });
+
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await other;
+    });
+
+    it('puts a call that never started on Puter, not on its handler', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const never = transport.send({ ...CALL, timeoutMs: 20 });
+        const queueTimedOut = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 1, status: 503, dispatch: 'queue-timeout' });
+
+        await expect(queueTimedOut).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: queue-timeout (503)',
+            dispatchReason: 'queue-timeout',
+        });
+        // Its wait plus the grace for the started line to arrive.
+        await expect(never).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: call never started',
+        });
+        const client = new EventsWorkerInvokerClient({} as IConfig);
+        client.setTransport({ send: async () => queueTimedOut });
+        await expect(client.invoke(REQUEST)).resolves.toMatchObject({
+            outcome: 'unavailable',
+        });
+    });
+
+    it('puts a call the stream ended without answering on Puter', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const answered = transport.send(CALL);
+        const dropped = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].end();
+
+        await expect(answered).resolves.toEqual({ status: 200, handled: true });
+        await expect(dropped).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: batch ended without an answer',
+        });
+    });
+
+    it('carries a dispatch failure in a line as that call`s reason', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        const missing = transport.send(CALL);
+        const other = transport.send(CALL);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+
+        await expect(missing).resolves.toEqual({
+            status: null,
+            error: 'dispatcher: missing (404)',
+            dispatchReason: 'missing',
+        });
+        await expect(other).resolves.toMatchObject({ status: 200 });
+    });
+
+    it('fails every call of a batch the dispatcher refused, or something else answered', async () => {
+        for (const [response, expected] of [
+            [
+                new Response('forbidden', {
+                    status: 403,
+                    headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'forbidden' },
+                }),
+                {
+                    status: null,
+                    error: 'dispatcher: forbidden (403)',
+                    dispatchReason: 'forbidden',
+                },
+            ],
+            [
+                new Response('not found', { status: 404 }),
+                {
+                    status: null,
+                    error: 'dispatcher: unmarked 404 for a batch',
+                },
+            ],
+        ] as const) {
+            const transport = transportFor(
+                (async () => response) as unknown as FetchImpl,
+            );
+            const answers = await Promise.all([
+                transport.send(CALL),
+                transport.send(CALL),
+            ]);
+            expect(answers).toEqual([expected, expected]);
+        }
+    });
+
+    it('fails every call when the batch request itself fails', async () => {
+        const transport = transportFor(refusedFetch);
+        const answers = await Promise.all([
+            transport.send(CALL),
+            transport.send(CALL),
+        ]);
+        expect(answers).toEqual([
+            { status: null, error: 'fetch failed' },
+            { status: null, error: 'fetch failed' },
+        ]);
+    });
+
+    it('falls back to single calls on a dispatcher without the batch route', async () => {
+        const urls: string[] = [];
+        const fetchImpl = (async (url: string) => {
+            urls.push(url);
+            if (url.endsWith(EVENTS_DISPATCH_BATCH_PATH))
+                return new Response('no-such-route', {
+                    status: 404,
+                    headers: {
+                        [EVENTS_DISPATCH_ERROR_HEADER]: 'no-such-route',
+                    },
+                });
+            return new Response(null, {
+                status: 200,
+                headers: { [EVENTS_HANDLED_HEADER]: '1' },
+            });
+        }) as unknown as FetchImpl;
+        const transport = transportFor(fetchImpl);
+
+        const first = await Promise.all([
+            transport.send(CALL),
+            transport.send(CALL),
+        ]);
+        expect(first).toEqual([
+            { status: 200, handled: true },
+            { status: 200, handled: true },
+        ]);
+        // The next pair goes straight to the single route.
+        await Promise.all([transport.send(CALL), transport.send(CALL)]);
+        expect(urls.map((url) => new URL(url).pathname)).toEqual([
+            EVENTS_DISPATCH_BATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+            EVENTS_DISPATCH_PATH,
+        ]);
+    });
+
+    it('builds each body when the batch goes out', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const transport = transportFor(fetchImpl);
+        let renders = 0;
+        const sent = [
+            transport.send({ ...CALL, render: () => `{"n":${++renders}}` }),
+            transport.send(CALL),
+        ];
+        expect(renders).toBe(0);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        expect(batches[0].body.calls[0].body).toBe('{"n":1}');
+        batches[0].answer({ i: 0, status: 200, handled: true });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await Promise.all(sent);
+    });
+
+    it('deploys a call`s missing script and retries it, batched with the rest', async () => {
+        const { fetchImpl, batches } = streamingFetch();
+        const client = new EventsWorkerInvokerClient({
+            events: { invokeTimeoutMs: 5_000 },
+        } as unknown as IConfig);
+        client.setTransport(transportFor(fetchImpl));
+        const missHandler = vi.fn(async () => 'deployed' as const);
+        client.setMissHandler(missHandler);
+
+        const missing = client.invoke({ ...REQUEST, script: 'evw-gone' });
+        const resident = client.invoke(REQUEST);
+        await vi.waitFor(() => expect(batches).toHaveLength(1));
+        batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
+        batches[0].answer({ i: 1, status: 200, handled: true });
+        await expect(resident).resolves.toMatchObject({ outcome: 'settled' });
+
+        // The retry is alone, so it goes on the single route.
+        await expect(missing).resolves.toMatchObject({ outcome: 'settled' });
+        expect(missHandler).toHaveBeenCalledWith('app-test', 'evw-gone', {
+            lane: undefined,
+        });
+    });
+
+    it('reads only well-formed answer lines', () => {
+        expect(parseAnswerLine('not json')).toBeNull();
+        expect(parseAnswerLine('{"status":200}')).toBeNull();
+        expect(parseAnswerLine('{"i":-1,"status":200}')).toBeNull();
+        expect(parseAnswerLine('{"i":0}')).toBeNull();
+        expect(parseAnswerLine('{"i":1,"started":true}')).toEqual({
+            i: 1,
+            started: true,
+        });
+        expect(parseAnswerLine('{"i":2,"status":200}')).toEqual({
+            i: 2,
+            answer: { status: 200, handled: false },
+        });
+    });
+});
+
+describe('EventsWorkerInvokerClient invoke options', () => {
+    it('stamps the event as each attempt goes out', async () => {
+        const bodies: string[] = [];
+        const client = makeClient();
+        client.setTransport({
+            send: async (call) => {
+                bodies.push(call.body);
+                return { status: 200, handled: true };
+            },
+        });
+        let n = 0;
+        await client.invoke(REQUEST, {
+            stampEvent: (event) => ({ ...(event as object), sentAt: ++n }),
+        });
+        expect(JSON.parse(bodies[0]).event).toEqual({ sentAt: 1 });
+    });
+
+    it('gives a call up as Puter`s when its deploy is put off for another', async () => {
+        const headersSeen: Array<Record<string, string>> = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch(headersSeen),
+            }),
+        );
+        client.setMissHandler(async () => 'busy');
+        await expect(client.invoke(REQUEST)).resolves.toMatchObject({
+            outcome: 'unavailable',
+            deployBusy: true,
+        });
+        expect(headersSeen).toHaveLength(1);
+    });
+
+    it('tells the miss handler whose call needs the deploy', async () => {
+        const lanes: unknown[] = [];
+        const client = makeClient();
+        client.setTransport(
+            new DispatcherInvokeTransport('http://dispatcher', 's', {
+                fetchImpl: missUntilDeployedFetch([]),
+            }),
+        );
+        client.setMissHandler(async (_appUid, _script, options) => {
+            lanes.push(options.lane);
+            return 'deployed';
+        });
+        await client.invoke(REQUEST, { lane: 'schedule' });
+        await client.invoke(REQUEST);
+        expect(lanes).toEqual(['schedule', undefined]);
+    });
+});
+
 const REQUEST: WorkerInvokeRequest = {
     script: 'evw-test',
     appUid: 'app-test',
@@ -313,7 +883,8 @@ const REQUEST: WorkerInvokeRequest = {
     ctx: {},
 };
 
-const makeClient = () => new EventsWorkerInvokerClient({} as unknown as IConfig);
+const makeClient = () =>
+    new EventsWorkerInvokerClient({} as unknown as IConfig);
 
 /** Answers `missing` until it sees the deployed header, then settles. */
 const missUntilDeployedFetch = (

src/backend/clients/events/EventsWorkerInvokerClient.ts

@@ -43,6 +43,20 @@ import { PuterClient } from '../types.js';
  *
  * On a miss, a handler set here may deploy the script itself and retry once
  * with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
+ *
+ * Calls that come due within a few milliseconds of each other go out together:
+ *
+ *     POST <events dispatcher>/invoke-batch
+ *     x-puter-internal-auth:  <internal secret>
+ *     { calls: [{ script, appUid, key, deployed?, body, waitMs }] }
+ *
+ * The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
+ * started: true }` when a call goes to its worker, then `{ i, status, handled?,
+ * error?, retryAfter?, dispatch? }` when it finishes — the same facts the
+ * single route carries in its status and headers. A call's timeout runs from
+ * its `started` line, so time spent queued behind other calls is never the
+ * handler's. A call still queued `waitMs` after the batch arrived is never
+ * started and answers `dispatch: "queue-timeout"`.
  */
 
 /** The one route an events worker answers. */
@@ -51,6 +65,27 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
 /** The route the events dispatcher takes an invocation on. */
 export const EVENTS_DISPATCH_PATH = '/invoke';
 
+/** The route it takes several on, answering each as it finishes. */
+export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
+
+/** Calls one batch carries at most. */
+export const INVOKE_BATCH_SIZE = 60;
+
+/** How long the first call of a batch waits for others to join it. */
+export const INVOKE_BATCH_WINDOW_MS = 5;
+
+/**
+ * How long a dispatcher without the batch route is sent single calls before the
+ * batch route is tried again.
+ */
+const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
+
+/**
+ * How long past its wait a batched call's `started` line may take to arrive
+ * before the call is given up as never started.
+ */
+const BATCH_START_GRACE_MS = 2_000;
+
 /**
  * Header the dispatcher marks its own failures with. Anything carrying it never
  * reached a handler, so its status says nothing about the delivery.
@@ -112,10 +147,16 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
  */
 const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
 
+/** Whose call asked for the deploy, for the share of deploys it may take. */
+export interface WorkerMissOptions {
+    lane?: InvokeLane;
+}
+
 /** What a self-deploy attempt did, and to which app/script it applies. */
 export type WorkerMissHandler = (
     appUid: string,
     script: string,
+    options: WorkerMissOptions,
 ) => Promise<'deployed' | string>;
 
 /** How long a handler has to answer before the attempt is abandoned. */
@@ -127,6 +168,18 @@ export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
  * that waits for a free one before its invoke timeout starts.
  */
 export const INVOKE_CONNECTIONS = 256;
+
+/**
+ * Of those, how many schedule runs may hold at once, so a burst of schedules
+ * can never take the connections event deliveries need.
+ */
+export const SCHEDULE_INVOKE_CONNECTIONS = 64;
+
+/** Whose call this is, for the connection share it may take. */
+export type InvokeLane = 'delivery' | 'schedule';
+
+/** What a deploy answers when another deploy of the region holds the way. */
+export const DEPLOY_BUSY = 'busy';
 const INVOKE_KEEP_ALIVE_MS = 30_000;
 
 /**
@@ -161,6 +214,12 @@ export interface EventsInvokeCall {
     timeoutMs: number;
     /** Set on the retry after this backend deployed the script itself. */
     deployed?: boolean;
+    lane?: InvokeLane;
+    /**
+     * Builds `body` again at send time, for a transport that waits for a
+     * connection first, so a send time stamped into it is the real one.
+     */
+    render?: () => string;
 }
 
 /** What one attempt at delivering an invocation came back with. */
@@ -176,6 +235,8 @@ export interface EventsInvokeAnswer {
     timedOut?: boolean;
     /** From a `retry-after` header in seconds. */
     retryAfterMs?: number;
+    /** A deploy this call needed was put off: another held the way. */
+    deployBusy?: boolean;
 }
 
 /**
@@ -194,10 +255,15 @@ export interface EventsInvokeTransport {
  * handler); retried like `retriable` but never the handler's fault.
  */
 export type WorkerInvokeOutcome =
-    'settled' | 'terminal' | 'retriable' | 'unavailable';
+    | 'settled'
+    | 'terminal'
+    | 'retriable'
+    | 'unavailable';
 
 export interface WorkerInvokeResult {
     outcome: WorkerInvokeOutcome;
+    /** A deploy on miss was put off because another one held the way. */
+    deployBusy?: boolean;
     /** The status the handler answered with, or `null` when it never did. */
     status: number | null;
     /** Why it could not answer, when nothing did. */
@@ -206,6 +272,13 @@ export interface WorkerInvokeResult {
     retryAfterMs?: number;
 }
 
+/** How one invocation is sent. */
+export interface WorkerInvokeOptions {
+    lane?: InvokeLane;
+    /** Applied to the event as each attempt is sent. */
+    stampEvent?: (event: unknown) => unknown;
+}
+
 const NO_TRANSPORT =
     'no events dispatcher configured (url and internal secret)';
 
@@ -236,7 +309,10 @@ export class EventsWorkerInvokerClient extends PuterClient {
         this.#dispatcher = null;
     }
 
-    async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
+    async invoke(
+        request: WorkerInvokeRequest,
+        options: WorkerInvokeOptions = {},
+    ): Promise<WorkerInvokeResult> {
         const transport = this.#pickTransport();
         if (!transport)
             return {
@@ -245,32 +321,41 @@ export class EventsWorkerInvokerClient extends PuterClient {
                 error: NO_TRANSPORT,
             };
 
-        const call: EventsInvokeCall = {
-            script: request.script,
-            appUid: request.appUid,
-            key: request.key,
-            body: JSON.stringify({
+        const { stampEvent } = options;
+        const render = (): string =>
+            JSON.stringify({
                 handler: request.handler,
-                event: request.event,
+                event: stampEvent ? stampEvent(request.event) : request.event,
                 ctx: request.ctx,
                 token: request.token,
-            }),
-            timeoutMs: this.#timeoutMs(),
-        };
+            });
+        const send = (deployed: boolean): Promise<EventsInvokeAnswer> =>
+            transport.send({
+                script: request.script,
+                appUid: request.appUid,
+                key: request.key,
+                body: render(),
+                timeoutMs: this.#timeoutMs(),
+                ...(deployed ? { deployed: true } : {}),
+                ...(options.lane ? { lane: options.lane } : {}),
+                ...(stampEvent ? { render } : {}),
+            });
 
-        let result = await transport.send(call);
+        let result = await send(false);
         if (this.#missHandler && selfDeployable(result)) {
             const outcome = await this.#missHandler(
                 request.appUid,
                 request.script,
+                { lane: options.lane },
             );
             if (outcome !== 'deployed')
                 return {
                     outcome: 'unavailable',
                     status: null,
                     error: `deploy: ${outcome}`,
+                    ...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
                 };
-            result = await transport.send({ ...call, deployed: true });
+            result = await send(true);
         } else if (
             result.status === null &&
             result.dispatchReason &&
@@ -278,10 +363,17 @@ export class EventsWorkerInvokerClient extends PuterClient {
         ) {
             // No deploy to trigger — just give propagation the one retry
             // it usually needs, without spending an upload on it.
-            result = await transport.send({ ...call, deployed: true });
+            result = await send(true);
         }
 
         const { status, handled, error, timedOut, retryAfterMs } = result;
+        if (result.deployBusy)
+            return {
+                outcome: 'unavailable',
+                status: null,
+                error,
+                deployBusy: true,
+            };
         // Past the timeout the handler was most likely still running, which
         // is its own doing; any other missing answer never reached it.
         if (status === null)
@@ -313,9 +405,19 @@ export class EventsWorkerInvokerClient extends PuterClient {
         // a misconfiguration worth reporting once per attempt rather than a
         // round trip.
         if (!dispatcherUrl || !internalSecret) return null;
+        const { invokeBatchSize, invokeBatchWindowMs } =
+            this.config.events ?? {};
         this.#dispatcher ??= new DispatcherInvokeTransport(
             dispatcherUrl,
             internalSecret,
+            {
+                ...(invokeBatchSize !== undefined
+                    ? { batchSize: invokeBatchSize }
+                    : {}),
+                ...(invokeBatchWindowMs !== undefined
+                    ? { batchWindowMs: invokeBatchWindowMs }
+                    : {}),
+            },
         );
         return this.#dispatcher;
     }
@@ -328,28 +430,63 @@ export class EventsWorkerInvokerClient extends PuterClient {
     }
 }
 
+/** A call waiting in, or sent with, the current batch. */
+interface BatchedCall {
+    call: EventsInvokeCall;
+    resolve: (answer: EventsInvokeAnswer) => void;
+    /** When its wait for a connection began, for what is left of it. */
+    waitingSince: number;
+}
+
+export interface DispatcherInvokeTransportOptions {
+    fetchImpl?: typeof undiciFetch;
+    /** Calls per batch; 1 sends every call on its own. */
+    batchSize?: number;
+    batchWindowMs?: number;
+}
+
 /** Reaches an events worker the only way production can: through the dispatcher. */
 export class DispatcherInvokeTransport implements EventsInvokeTransport {
     readonly #url: string;
     readonly #secret: string;
     readonly #fetchImpl: typeof undiciFetch;
+    readonly #batchSize: number;
+    readonly #batchWindowMs: number;
     #agent: Agent | null = null;
     /** Calls holding a connection slot. */
     #active = 0;
-    /** Calls waiting for one, longest-waiting first. */
-    readonly #waiting = new Set<() => void>();
+    /** Of those, schedule runs. */
+    #scheduleActive = 0;
+    /** Calls waiting for one, longest-waiting first, with their lane. */
+    readonly #waiting = new Map<() => void, InvokeLane>();
+    /** Calls gathered for the next batch. */
+    #batch: BatchedCall[] = [];
+    #batchTimer: ReturnType<typeof setTimeout> | null = null;
+    /** Until when the dispatcher is taken not to have the batch route. */
+    #batchUnsupportedUntil = 0;
 
     constructor(
         dispatcherUrl: string,
         secret: string,
-        options: { fetchImpl?: typeof undiciFetch } = {},
+        options: DispatcherInvokeTransportOptions = {},
     ) {
-        this.#url = dispatcherUrl;
+        // Joined as a string: `new URL(path, base)` resolves an absolute path
+        // against the base's origin, dropping any path prefix it carries.
+        this.#url = dispatcherUrl.replace(/\/$/, '');
         this.#secret = secret;
         this.#fetchImpl = options.fetchImpl ?? undiciFetch;
+        this.#batchSize = Math.max(
+            1,
+            Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
+        );
+        this.#batchWindowMs = Math.max(
+            0,
+            options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
+        );
     }
 
     close(): void {
+        this.#flush();
         void this.#agent?.close().catch(() => {});
         this.#agent = null;
     }
@@ -357,29 +494,68 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
         // No `timedOut`: the handler never got this call, so the wait is not
         // its failure.
-        if (!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE)))
+        const lane = call.lane ?? 'delivery';
+        const waitingSince = Date.now();
+        if (
+            !(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
+        )
             return {
                 status: null,
                 error: 'no free connection to the events dispatcher',
             };
         try {
-            // `new URL(path, base)` resolves an absolute path against the
-            // base's origin, dropping any path prefix `base` already carries.
-            const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
-            const response = await this.#fetchImpl(url, {
-                method: 'POST',
-                headers: {
-                    'content-type': 'application/json',
-                    'x-puter-internal-auth': this.#secret,
-                    'x-puter-events-script': call.script,
-                    'x-puter-events-app': call.appUid,
-                    'x-puter-events-key': call.key,
-                    ...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
-                },
-                body: call.body,
-                signal: AbortSignal.timeout(call.timeoutMs),
-                dispatcher: this.#dispatcherFor(),
+            if (this.#batchSize === 1) return await this.#sendOne(call);
+            return await new Promise<EventsInvokeAnswer>((resolve) => {
+                this.#batch.push({ call, resolve, waitingSince });
+                if (this.#batch.length >= this.#batchSize) this.#flush();
+                else
+                    this.#batchTimer ??= setTimeout(
+                        () => this.#flush(),
+                        this.#batchWindowMs,
+                    );
             });
+        } finally {
+            this.#release(lane);
+        }
+    }
+
+    /** Send what has gathered: alone when it is one call, else as a batch. */
+    #flush(): void {
+        if (this.#batchTimer) clearTimeout(this.#batchTimer);
+        this.#batchTimer = null;
+        const batch = this.#batch;
+        this.#batch = [];
+        if (batch.length === 0) return;
+        if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
+            for (const { call, resolve } of batch)
+                void this.#sendOne(call).then(resolve);
+            return;
+        }
+        void this.#sendBatch(batch);
+    }
+
+    /** One call on the single route. */
+    async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                        'x-puter-events-script': call.script,
+                        'x-puter-events-app': call.appUid,
+                        'x-puter-events-key': call.key,
+                        ...(call.deployed
+                            ? { [EVENTS_DEPLOYED_HEADER]: '1' }
+                            : {}),
+                    },
+                    body: call.render ? call.render() : call.body,
+                    signal: AbortSignal.timeout(call.timeoutMs),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
             // The body is not read: a handler answers with a status, and
             // whatever else it writes is its own business.
             void response.body?.cancel().catch(() => {});
@@ -390,24 +566,181 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 EVENTS_DISPATCH_ERROR_HEADER,
             );
             if (dispatchError !== null)
-                return {
-                    status: null,
-                    error: `dispatcher: ${dispatchError} (${response.status})`,
-                    dispatchReason: dispatchError,
-                };
+                return dispatchAnswer(dispatchError, response.status);
 
             return answerFrom(response);
         } catch (err) {
             return failedAnswer(err);
-        } finally {
-            this.#release();
         }
     }
 
+    /**
+     * Several calls in one request. A call may wait in the dispatcher's queue
+     * for whatever is left of its connection wait, and its timeout runs from
+     * its `started` line; the request itself is dropped once every call has an
+     * answer.
+     */
+    async #sendBatch(batch: BatchedCall[]): Promise<void> {
+        const hangUp = new AbortController();
+        let open = batch.length;
+        const answered = batch.map(() => false);
+        const timers: ReturnType<typeof setTimeout>[] = [];
+        const settle = (i: number, answer: EventsInvokeAnswer): void => {
+            if (answered[i]) return;
+            answered[i] = true;
+            clearTimeout(timers[i]);
+            batch[i].resolve(answer);
+            if (--open === 0) hangUp.abort();
+        };
+        const settleRest = (answer: EventsInvokeAnswer): void => {
+            for (let i = 0; i < batch.length; i++) settle(i, answer);
+        };
+
+        const now = Date.now();
+        const waits = batch.map(({ call, waitingSince }) =>
+            Math.max(
+                0,
+                Math.floor(
+                    call.timeoutMs * CONNECTION_WAIT_SHARE -
+                        (now - waitingSince),
+                ),
+            ),
+        );
+        const body = JSON.stringify({
+            calls: batch.map(({ call }, i) => ({
+                script: call.script,
+                appUid: call.appUid,
+                key: call.key,
+                ...(call.deployed ? { deployed: true } : {}),
+                body: call.render ? call.render() : call.body,
+                waitMs: waits[i],
+            })),
+        });
+        const arm = (
+            i: number,
+            ms: number,
+            answer: () => EventsInvokeAnswer,
+        ) => {
+            clearTimeout(timers[i]);
+            timers[i] = setTimeout(() => settle(i, answer()), ms);
+            timers[i].unref?.();
+        };
+        // No `timedOut`: a call that never started never reached a handler.
+        batch.forEach((_, i) =>
+            arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
+                status: null,
+                error: 'dispatcher: call never started',
+            })),
+        );
+        const started = (i: number): void => {
+            if (!answered[i])
+                arm(i, batch[i].call.timeoutMs, () =>
+                    failedAnswer(timeoutError()),
+                );
+        };
+        const longest = Math.max(
+            ...batch.map(
+                ({ call }, i) =>
+                    waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
+            ),
+        );
+
+        try {
+            const response = await this.#fetchImpl(
+                `${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        'x-puter-internal-auth': this.#secret,
+                    },
+                    body,
+                    signal: AbortSignal.any([
+                        hangUp.signal,
+                        AbortSignal.timeout(longest),
+                    ]),
+                    dispatcher: this.#dispatcherFor(),
+                },
+            );
+
+            const dispatchError = response.headers.get(
+                EVENTS_DISPATCH_ERROR_HEADER,
+            );
+            if (dispatchError === 'no-such-route') {
+                // A dispatcher from before the batch route: nothing ran, so
+                // every call goes again on its own.
+                void response.body?.cancel().catch(() => {});
+                this.#batchUnsupportedUntil =
+                    Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
+                batch.forEach(({ call }, i) => {
+                    if (!answered[i])
+                        void this.#sendOne(call).then((a) => settle(i, a));
+                });
+                return;
+            }
+            if (dispatchError !== null || !response.ok) {
+                void response.body?.cancel().catch(() => {});
+                // An unmarked failure (an edge 404, a WAF 403) reads as
+                // Puter's, never as any handler's answer.
+                settleRest(
+                    dispatchError !== null
+                        ? dispatchAnswer(dispatchError, response.status)
+                        : {
+                              status: null,
+                              error: `dispatcher: unmarked ${response.status} for a batch`,
+                          },
+                );
+                return;
+            }
+
+            let buffered = '';
+            const read = (line: string): void => {
+                if (line.trim() === '') return;
+                const parsed = parseAnswerLine(line);
+                if (!parsed || parsed.i >= batch.length) return;
+                if ('started' in parsed) started(parsed.i);
+                else settle(parsed.i, parsed.answer);
+            };
+            const decoder = new TextDecoder();
+            for await (const chunk of response.body ?? []) {
+                buffered += decoder.decode(chunk, { stream: true });
+                let newline = buffered.indexOf('\n');
+                while (newline !== -1) {
+                    read(buffered.slice(0, newline));
+                    buffered = buffered.slice(newline + 1);
+                    newline = buffered.indexOf('\n');
+                }
+            }
+            read(buffered);
+            settleRest({
+                status: null,
+                error: 'dispatcher: batch ended without an answer',
+            });
+        } catch (err) {
+            // Every call has its answer and the request was dropped on purpose.
+            if (open === 0) return;
+            settleRest(failedAnswer(err));
+        }
+    }
+
+    /** Whether a call on `lane` may take a connection now. */
+    #free(lane: InvokeLane): boolean {
+        return (
+            this.#active < INVOKE_CONNECTIONS &&
+            (lane !== 'schedule' ||
+                this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
+        );
+    }
+
+    #take(lane: InvokeLane): void {
+        this.#active++;
+        if (lane === 'schedule') this.#scheduleActive++;
+    }
+
     /** A connection slot, or false once `waitMs` passes without one. */
-    #acquire(waitMs: number): Promise<boolean> {
-        if (this.#active < INVOKE_CONNECTIONS) {
-            this.#active++;
+    #acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
+        if (this.#free(lane)) {
+            this.#take(lane);
             return Promise.resolve(true);
         }
         return new Promise((resolve) => {
@@ -420,19 +753,24 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
                 resolve(false);
             }, waitMs);
             timer.unref?.();
-            this.#waiting.add(granted);
+            this.#waiting.set(granted, lane);
         });
     }
 
-    /** Hand the slot to the longest waiter, or give it back. */
-    #release(): void {
-        const next = this.#waiting.values().next().value;
-        if (next) {
-            this.#waiting.delete(next);
-            next();
+    /**
+     * Give the slot back, then hand one to the longest waiter it can go to: a
+     * schedule waiter at a full schedule share doesn't hold up a delivery.
+     */
+    #release(lane: InvokeLane): void {
+        this.#active--;
+        if (lane === 'schedule') this.#scheduleActive--;
+        for (const [granted, waiting] of this.#waiting) {
+            if (!this.#free(waiting)) continue;
+            this.#waiting.delete(granted);
+            this.#take(waiting);
+            granted();
             return;
         }
-        this.#active--;
     }
 
     /** One pool per process, built on first use and closed with the server. */
@@ -446,6 +784,65 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
     }
 }
 
+/** What a failure the dispatcher marked as its own reads as. */
+const dispatchAnswer = (
+    reason: string,
+    status: number,
+): EventsInvokeAnswer => ({
+    status: null,
+    error: `dispatcher: ${reason} (${status})`,
+    dispatchReason: reason,
+});
+
+/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
+const timeoutError = (): Error =>
+    new DOMException(
+        'The operation was aborted due to timeout',
+        'TimeoutError',
+    );
+
+/** One line of a batch answer, or null when it is not one. */
+export const parseAnswerLine = (
+    line: string,
+):
+    | { i: number; started: true }
+    | { i: number; answer: EventsInvokeAnswer }
+    | null => {
+    let raw: unknown;
+    try {
+        raw = JSON.parse(line);
+    } catch {
+        return null;
+    }
+    if (!raw || typeof raw !== 'object') return null;
+    const { i, started, status, handled, error, retryAfter, dispatch } =
+        raw as Record<string, unknown>;
+    if (!Number.isInteger(i) || (i as number) < 0) return null;
+    if (started === true) return { i: i as number, started: true };
+    if (typeof status !== 'number') return null;
+    if (typeof dispatch === 'string')
+        return { i: i as number, answer: dispatchAnswer(dispatch, status) };
+    return {
+        i: i as number,
+        answer: answerFrom({
+            status,
+            headers: {
+                get: (name) => {
+                    if (name === EVENTS_HANDLED_HEADER)
+                        return handled === true ? '1' : null;
+                    if (name === EVENTS_ERROR_HEADER)
+                        return typeof error === 'string' ? error : null;
+                    if (name === 'retry-after')
+                        return typeof retryAfter === 'string'
+                            ? retryAfter
+                            : null;
+                    return null;
+                },
+            },
+        }),
+    };
+};
+
 /** Whether deploying the script is what fixes this answer. */
 const selfDeployable = (result: EventsInvokeAnswer): boolean =>
     result.status === null

src/backend/controllers/broadcast/BroadcastController.ts

@@ -19,6 +19,7 @@
 
 import type { Request, Response } from 'express';
 import { Controller, Post } from '../../core/http/decorators.js';
+import { schedulesUnavailable } from '../../core/schedules.js';
 import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
 import type { ForwardBatch } from '../../services/events/forwardQueue.js';
 import { PuterController } from '../types.js';
@@ -108,6 +109,42 @@ export class BroadcastController extends PuterController {
         );
         res.status(200).json({ ok: true, ...reply });
     }
+
+    /**
+     * Schedule changes a region made for schedules this one owns, handed to
+     * `services.schedules`; `events_disabled` on a server without it.
+     */
+    @Post('/events/schedules', { subdomain: '*' })
+    async schedules(req: Request, res: Response): Promise<void> {
+        const broadcast = this.services.broadcast as unknown as
+            BroadcastService | undefined;
+        if (!broadcast) {
+            res.status(503).json({
+                error: { message: 'Broadcast service not registered' },
+            });
+            return;
+        }
+
+        const verified = await broadcast.verifySignedRequest(
+            req.rawBody,
+            broadcastHeaders(req),
+        );
+        if (!verified.ok) {
+            res.status(verified.status ?? 400).json({
+                error: { message: verified.message ?? 'Bad request' },
+            });
+            return;
+        }
+        if (verified.info?.ignored) {
+            res.status(200).json({ ok: true, ...verified.info });
+            return;
+        }
+
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        const reply = await schedules.receivePeerChanges(req.body);
+        res.status(200).json({ ok: true, ...reply });
+    }
 }
 
 const broadcastHeaders = (req: Request) => {

src/backend/controllers/events/EventsController.ts

@@ -25,6 +25,11 @@ import { bodyRecord } from '../../core/http/requestBody.js';
 import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
 import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
 import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    schedulesUnavailable,
+    type ScheduleServiceContract,
+} from '../../core/schedules.js';
 import { normalizeLimit } from '../../util/pagination.js';
 import { PuterController } from '../types.js';
 import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -169,6 +174,88 @@ export class EventsController extends PuterController {
         res.json({});
     }
 
+    // -- Schedules ---------------------------------------------------
+    //
+    // An app's handler run on a timer, served by `services.schedules` when a
+    // server has it and answered `events_disabled` when it doesn't. Creating
+    // one needs an app context; listing, reading and removing are scoped the
+    // way the durable verbs are. The create and remove verbs spend the
+    // subscribe and unsubscribe budgets in the service, shared with those
+    // verbs.
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /** POST /events/schedules — create a schedule for the calling app. */
+    @Post('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async createSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
+    }
+
+    /** GET /events/schedules — the caller's schedules, one page at a time. */
+    @Get('/schedules', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async listSchedules(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        const query = (req.query ?? {}) as Record<string, unknown>;
+
+        const page = await this.#schedules().list(actor, {
+            limit: normalizeLimit(query.limit, {
+                cap: SCHEDULE_LIST_LIMIT_CAP,
+            }),
+            cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
+            includeTotal: query.includeTotal === 'true',
+        });
+
+        res.json({
+            items: page.items,
+            ...(page.cursor ? { cursor: page.cursor } : {}),
+            ...(page.total !== undefined ? { total: page.total } : {}),
+        });
+    }
+
+    /**
+     * POST /events/schedules/remove — end one schedule. An id the caller does
+     * not hold reads as absent.
+     */
+    @Post('/schedules/remove', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+    })
+    async removeSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
+        res.json({});
+    }
+
+    /**
+     * GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
+     * caller does not hold reads as absent.
+     */
+    @Get('/schedules/:scheduleId', {
+        subdomain: 'api',
+        requireAuth: true,
+        allowAccessToken: true,
+        rateLimit: EVENTS_LIST_LIMIT,
+    })
+    async getSchedule(req: Request, res: Response): Promise<void> {
+        const actor = this.#requireActor(req);
+        res.json(await this.#schedules().get(actor, req.params.scheduleId));
+    }
+
     // -- Cross-user key-value handles --------------------------------
 
     /**
@@ -425,10 +512,14 @@ export class EventsController extends PuterController {
             res.json({ deployed: true });
             return;
         }
-        // `failed` and `throttled` are worth another attempt: the rest say
-        // this script is not what the app's handlers currently are.
+        // `failed`, `throttled` and `busy` are worth another attempt: the rest
+        // say this script is not what the app's handlers currently are.
         res.status(
-            outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
+            outcome === 'failed' ||
+                outcome === 'throttled' ||
+                outcome === 'busy'
+                ? 502
+                : 404,
         ).json({
             deployed: false,
             reason: outcome,
@@ -455,8 +546,8 @@ export class EventsController extends PuterController {
             );
             return;
         }
-        this.services.events.useWorkerMissHandler((appUid, script) =>
-            deployer.ensure(appUid, script),
+        this.services.events.useWorkerMissHandler((appUid, script, options) =>
+            deployer.ensure(appUid, script, options),
         );
     }
 

src/backend/controllers/events/limits.test.ts

@@ -41,16 +41,30 @@ import {
     EVENTS_SINGLE_DELIVERY_LIMIT,
     EVENTS_WORKER_INVOCATION_LIMIT,
     limitFor,
+    tiered,
     type TieredLimit,
 } from './limits.js';
 
 const tiers: Array<[string, TieredLimit]> = [
-    ['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
+    [
+        'durable subscriptions per account',
+        EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
+    ],
     ['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
     ['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
     ['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
 ];
 
+describe('tiered', () => {
+    it('gives paid plans the base, and the free and temporary tiers their own', () => {
+        const limit = tiered(30, 20, 10);
+        expect(limitFor(limit, null)).toBe(30);
+        expect(limitFor(limit, 'some_plan')).toBe(30);
+        expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
+        expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
+    });
+});
+
 describe('the tiered subscription quotas', () => {
     it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
         for (const n of Object.values(tier.bySubscription))
@@ -105,9 +119,9 @@ describe('the tiered subscription quotas', () => {
     });
 
     it('holds an unrecognised plan to the paid base', () => {
-        expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
-            EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
-        );
+        expect(
+            limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
+        ).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
         expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
             EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
         );
@@ -131,9 +145,9 @@ describe('the delivery budgets', () => {
             expect(
                 limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
             ).toBe(matched);
-            expect(
-                limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
-            ).toBe(evaluated);
+            expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
+                evaluated,
+            );
             expect(evaluated).toBeGreaterThanOrEqual(matched);
         }
     });

src/backend/controllers/events/limits.ts

@@ -53,7 +53,12 @@ export interface TieredLimit {
     bySubscription: Record<string, number>;
 }
 
-const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
+/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
+export const tiered = (
+    paid: number,
+    free: number,
+    temp: number,
+): TieredLimit => ({
     limit: paid,
     bySubscription: {
         [DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -448,11 +453,13 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
 // finish backs off the same way but is not the handler's failure, so it never
 // counts toward that run.
 
-/** Wait before the first retry of a delivery a handler could not take. */
-export const EVENTS_RETRY_BASE_MS = 2_000;
-
-/** Longest wait between retries, however many have failed. */
-export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+// Wait before the first retry of a delivery a handler could not take, the
+// longest wait, and the doubling between them. Shared with system jobs.
+export {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from '../../util/retryBackoff.js';
 
 /**
  * Handler failures in a row before a subscription is suspended. Counted per
@@ -468,13 +475,6 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
  */
 export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
 
-/** How long to hold a delivery whose handler has failed `attempts` times. */
-export const deliveryBackoffMs = (attempts: number): number =>
-    Math.min(
-        EVENTS_RETRY_MAX_MS,
-        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
-    );
-
 // -- Suspended backlog -----------------------------------------------
 //
 // A suspended subscription stops metering but keeps what it is owed, and that

src/backend/controllers/events/scheduleRoutes.test.ts

@@ -0,0 +1,274 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The schedule routes are thin delegates: without `services.schedules` every
+ * one answers 503 `events_disabled`; with it, each hands its request over and
+ * answers what it returns.
+ */
+
+import type { Request, Response } from 'express';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    ScheduleServiceContract,
+    ScheduleView,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { BroadcastController } from '../broadcast/BroadcastController.js';
+import type { EventsController } from './EventsController.js';
+
+let server: PuterServer;
+let events: EventsController;
+let broadcast: BroadcastController;
+let actor: Actor;
+let saved: unknown;
+
+const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
+
+const view = {
+    scheduleId: SCHEDULE_ID,
+    kind: 'recurring',
+    appUid: 'app-x',
+} as ScheduleView;
+
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const fakeSchedules = () =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(async () => view),
+        list: vi.fn(async () => ({
+            items: [view],
+            cursor: 'next',
+            total: 3,
+        })),
+        get: vi.fn(async () => view),
+        remove: vi.fn(async () => {}),
+        receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    }) satisfies ScheduleServiceContract;
+
+const req = (init: {
+    body?: unknown;
+    query?: Record<string, unknown>;
+    params?: Record<string, string>;
+}): Request =>
+    ({
+        actor,
+        body: init.body ?? {},
+        query: init.query ?? {},
+        params: init.params ?? {},
+        headers: {},
+        rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
+    }) as unknown as Request;
+
+const res = () => {
+    const sent: { status: number; body: unknown } = {
+        status: 200,
+        body: undefined,
+    };
+    const response = {
+        json: vi.fn((body: unknown) => {
+            sent.body = body;
+            return response;
+        }),
+        status: vi.fn((code: number) => {
+            sent.status = code;
+            return response;
+        }),
+        setHeader: vi.fn(() => response),
+    };
+    return { response: response as unknown as Response, sent };
+};
+
+/** Each route, called as the router would. */
+const routes: Array<[string, (response: Response) => Promise<void>]> = [
+    [
+        'POST /events/schedules',
+        (r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
+    ],
+    [
+        'GET /events/schedules',
+        (r) => events.listSchedules(req({ query: { limit: '5' } }), r),
+    ],
+    [
+        'GET /events/schedules/:scheduleId',
+        (r) =>
+            events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
+    ],
+    [
+        'POST /events/schedules/remove',
+        (r) =>
+            events.removeSchedule(
+                req({ body: { scheduleId: SCHEDULE_ID } }),
+                r,
+            ),
+    ],
+    [
+        'POST /broadcast/events/schedules',
+        (r) =>
+            broadcast.schedules(
+                req({ body: { namespace: 'n', changes: [] } }),
+                r,
+            ),
+    ],
+];
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    events = server.controllers.events as unknown as EventsController;
+    broadcast = server.controllers.broadcast as unknown as BroadcastController;
+    actor = makeActor({
+        user: { id: 1, uuid: 'u-1', username: 'someone' },
+    } as never);
+    saved = slot().schedules;
+    vi.spyOn(
+        server.services.broadcast,
+        'verifySignedRequest',
+    ).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+});
+
+afterAll(async () => {
+    vi.restoreAllMocks();
+    await server?.shutdown();
+});
+
+describe('without services.schedules', () => {
+    it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
+        setSchedules(undefined);
+        const { response } = res();
+        let thrown: unknown;
+        try {
+            await call(response);
+        } catch (err) {
+            thrown = err;
+        }
+        expect(isHttpError(thrown)).toBe(true);
+        expect(thrown).toMatchObject({
+            statusCode: 503,
+            legacyCode: 'events_disabled',
+            // The designed answer, so it never pages.
+            noAlarm: true,
+        });
+    });
+
+    it('refuses an unsigned peer batch before saying schedules are off', async () => {
+        setSchedules(undefined);
+        vi.spyOn(
+            server.services.broadcast,
+            'verifySignedRequest',
+        ).mockResolvedValueOnce({
+            ok: false,
+            status: 401,
+            message: 'bad signature',
+        } as never);
+        const { response, sent } = res();
+        await broadcast.schedules(
+            req({ body: { namespace: 'n', changes: [] } }),
+            response,
+        );
+        expect(sent.status).toBe(401);
+    });
+});
+
+describe('with services.schedules', () => {
+    it('creates through createFor and answers its view', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.createSchedule(req({ body: { every: 60_000 } }), response);
+        expect(schedules.createFor).toHaveBeenCalledWith(actor, {
+            every: 60_000,
+        });
+        expect(sent.body).toEqual(view);
+    });
+
+    it('lists with the limit capped, the cursor and includeTotal', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.listSchedules(
+            req({
+                query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
+            }),
+            response,
+        );
+        expect(schedules.list).toHaveBeenCalledWith(actor, {
+            limit: 200,
+            cursor: 'c',
+            includeTotal: true,
+        });
+        expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
+    });
+
+    it('reads one through get', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.getSchedule(
+            req({ params: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual(view);
+    });
+
+    it('removes through remove and answers {}', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        await events.removeSchedule(
+            req({ body: { scheduleId: SCHEDULE_ID } }),
+            response,
+        );
+        expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
+        expect(sent.body).toEqual({});
+    });
+
+    it('hands a signed peer batch to receivePeerChanges', async () => {
+        const schedules = fakeSchedules();
+        setSchedules(schedules);
+        const { response, sent } = res();
+        const body = { namespace: 'n', changes: [] };
+        await broadcast.schedules(req({ body }), response);
+        expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
+        expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
+    });
+});

src/backend/controllers/events/workerDeploy.test.ts

@@ -25,8 +25,13 @@
 
 import MockRedis from 'ioredis-mock';
 import { describe, expect, it } from 'vitest';
+import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
 import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
-import { EventsWorkerDeployer } from './workerDeploy.js';
+import {
+    EventsWorkerDeployer,
+    LocalEventsInvokeTransport,
+    type EventsDeployOutcome,
+} from './workerDeploy.js';
 import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
 import { handlerSetHash } from '../../services/events/workerSource.js';
 import type { IConfig } from '../../types.js';
@@ -176,3 +181,151 @@ describe('EventsWorkerDeployer.ensure', () => {
         expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
     });
 });
+
+/** A server's schedules with a deploy gate that records and can refuse. */
+const withGate = (
+    layers: Layers,
+    refuse: (capped: boolean) => boolean = () => false,
+) => {
+    const gated: Array<{ script: string; capped: boolean }> = [];
+    const gate = {
+        gateDeploy: async (
+            script: string,
+            options: { capped: boolean },
+            deploy: () => Promise<EventsDeployOutcome>,
+        ): Promise<EventsDeployOutcome> => {
+            gated.push({ script, capped: options.capped });
+            return refuse(options.capped) ? 'busy' : deploy();
+        },
+    };
+    const deployer = new EventsWorkerDeployer({
+        ...layers,
+        services: { ...layers.services, schedules: gate },
+    } as unknown as Layers);
+    return { deployer, gated };
+};
+
+describe('the schedules` deploy gate', () => {
+    it('is skipped without schedules, or with schedules that have no gate', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        await expect(
+            new EventsWorkerDeployer(layers).ensure('app-x', SCRIPT, {
+                lane: 'schedule',
+            }),
+        ).resolves.toBe('deployed');
+        const gateless = new EventsWorkerDeployer({
+            ...layers,
+            services: { ...layers.services, schedules: {} },
+        } as unknown as Layers);
+        await expect(
+            gateless.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('deployed');
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('takes every deploy, capped only for a schedule run', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers);
+
+        await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
+            'deployed',
+        );
+        await expect(
+            deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('deployed');
+        expect(gated).toEqual([
+            { script: SCRIPT, capped: false },
+            { script: SCRIPT, capped: true },
+        ]);
+        expect(created).toEqual([SCRIPT, SCRIPT]);
+    });
+
+    it('answers a refused schedule run busy, and a delivery still deploys', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer } = withGate(layers, (capped) => capped);
+
+        await expect(
+            deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
+        ).resolves.toBe('busy');
+        await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
+            'deployed',
+        );
+        expect(created).toEqual([SCRIPT]);
+    });
+
+    it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const scheduled = deployer.ensure('app-x', SCRIPT, {
+            lane: 'schedule',
+        });
+        const deliveries = [
+            deployer.ensure('app-x', SCRIPT),
+            deployer.ensure('app-x', SCRIPT),
+        ];
+        await expect(scheduled).resolves.toBe('busy');
+        await expect(Promise.all(deliveries)).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(created).toEqual([SCRIPT]);
+        expect(gated.map((g) => g.capped)).toEqual([true, false]);
+    });
+
+    it('lets a schedule run join a delivery`s deploy in flight', async () => {
+        const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
+        const { deployer, gated } = withGate(layers, (capped) => capped);
+
+        const delivery = deployer.ensure('app-x', SCRIPT);
+        const scheduled = deployer.ensure('app-x', SCRIPT, {
+            lane: 'schedule',
+        });
+        await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
+            'deployed',
+            'deployed',
+        ]);
+        expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
+        expect(created).toEqual([SCRIPT]);
+    });
+});
+
+describe('LocalEventsInvokeTransport', () => {
+    it('renders each request as it goes out, after a cold deploy', async () => {
+        const bodies: string[] = [];
+        let resident = false;
+        const transport = new LocalEventsInvokeTransport(
+            {
+                localworkerservice: {
+                    dispatchEventsWorker: async (request: Request) => {
+                        bodies.push(await request.text());
+                        if (!resident) return null;
+                        return new Response(null, {
+                            status: 200,
+                            headers: { 'x-puter-events-handled': '1' },
+                        });
+                    },
+                },
+            } as never,
+            {
+                ensure: async () => {
+                    resident = true;
+                    return 'deployed';
+                },
+            } as unknown as EventsWorkerDeployer,
+        );
+        let renders = 0;
+        const call: EventsInvokeCall = {
+            script: SCRIPT,
+            appUid: 'app-x',
+            key: 'k',
+            body: '{"n":0}',
+            timeoutMs: 5000,
+            render: () => `{"n":${++renders}}`,
+        };
+        await expect(transport.send(call)).resolves.toMatchObject({
+            status: 200,
+        });
+        expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
+    });
+});

src/backend/controllers/events/workerDeploy.ts

@@ -29,6 +29,7 @@ import {
 import type {
     EventsInvokeAnswer,
     EventsInvokeCall,
+    InvokeLane,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import {
     EVENTS_INVOKE_PATH,
@@ -36,6 +37,7 @@ import {
     failedAnswer,
 } from '../../clients/events/EventsWorkerInvokerClient.js';
 import type { puterServices } from '../../services/index.js';
+import type { IExtensionServiceInstances } from '../../services/types.js';
 import {
     eventsInvokeKey,
     eventsWorkerScope,
@@ -61,13 +63,25 @@ import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
 
 /** Why a deploy could not happen, for the callers that answer differently. */
 export type EventsDeployOutcome =
-    'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
+    | 'deployed'
+    | 'stale'
+    | 'no-handlers'
+    | 'no-owner'
+    | 'throttled'
+    | 'failed'
+    | 'busy';
+
+/** Whose call asked for the deploy: a schedule run's is held to a share. */
+export interface EventsDeployOptions {
+    lane?: InvokeLane;
+}
 
 interface DeployLayers {
     config: IConfig;
     clients: LayerInstances<typeof puterClients>;
     stores: LayerInstances<typeof puterStores>;
-    services: LayerInstances<typeof puterServices>;
+    services: LayerInstances<typeof puterServices> &
+        Pick<IExtensionServiceInstances, 'schedules'>;
     drivers: LayerInstances<typeof puterDrivers>;
 }
 
@@ -77,6 +91,12 @@ interface DeployLayers {
  */
 const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
 
+interface InFlightDeploy {
+    /** A schedule run's, which the region's deploy slots may refuse. */
+    capped: boolean;
+    promise: Promise<EventsDeployOutcome>;
+}
+
 interface DeployResult {
     success?: boolean;
     errors?: unknown[];
@@ -85,7 +105,7 @@ interface DeployResult {
 export class EventsWorkerDeployer {
     readonly #layers: DeployLayers;
     /** Collapses the burst of misses one cold app produces into one deploy. */
-    readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
+    readonly #inFlight = new Map<string, InFlightDeploy>();
 
     constructor(layers: DeployLayers) {
         this.#layers = layers;
@@ -104,15 +124,79 @@ export class EventsWorkerDeployer {
      * A request for anything else is `stale` — the set moved on, and the next
      * delivery resolves the new script rather than this one being revived.
      */
-    ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
+    ensure(
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions = {},
+    ): Promise<EventsDeployOutcome> {
         const key = `${appUid}|${script}`;
+        const capped = options.lane === 'schedule';
         const existing = this.#inFlight.get(key);
-        if (existing) return existing;
-        const started = this.#deploy(appUid, script).finally(() =>
-            this.#inFlight.delete(key),
+        if (!existing) return this.#start(key, appUid, script, options);
+        if (capped || !existing.capped) return existing.promise;
+        return this.#afterCapped(
+            existing.promise,
+            key,
+            appUid,
+            script,
+            options,
+        );
+    }
+
+    #start(
+        key: string,
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const entry: InFlightDeploy = {
+            capped: options.lane === 'schedule',
+            promise: this.#gated(appUid, script, options).finally(() => {
+                if (this.#inFlight.get(key) === entry)
+                    this.#inFlight.delete(key);
+            }),
+        };
+        this.#inFlight.set(key, entry);
+        return entry.promise;
+    }
+
+    /**
+     * A delivery that joined a schedule run's deploy: `busy` means no deploy
+     * started, and a delivery is never refused a slot, so it asks again.
+     */
+    async #afterCapped(
+        capped: Promise<EventsDeployOutcome>,
+        key: string,
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const outcome = await capped;
+        if (outcome !== 'busy') return outcome;
+        const current = this.#inFlight.get(key);
+        if (current && !current.capped) return current.promise;
+        return this.#start(key, appUid, script, options);
+    }
+
+    /**
+     * Every deploy goes through the schedules' deploy gate when a server has
+     * one: a per-script lock, so a burst of runs of an evicted script deploys
+     * it once, and a region-wide cap that a schedule run's deploy (`capped`)
+     * may be refused with `busy`. A delivery's deploy is never refused.
+     */
+    #gated(
+        appUid: string,
+        script: string,
+        options: EventsDeployOptions,
+    ): Promise<EventsDeployOutcome> {
+        const deploy = () => this.#deploy(appUid, script);
+        return (
+            this.#layers.services.schedules?.gateDeploy?.(
+                script,
+                { capped: options.lane === 'schedule' },
+                deploy,
+            ) ?? deploy()
         );
-        this.#inFlight.set(key, started);
-        return started;
     }
 
     async #deploy(
@@ -299,6 +383,8 @@ export class LocalEventsInvokeTransport {
     }
 
     async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
+        // Built per attempt, so a send time stamped into the body is taken
+        // after a cold deploy rather than before it.
         const request = (): Request =>
             new Request(
                 `http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -308,7 +394,7 @@ export class LocalEventsInvokeTransport {
                         'content-type': 'application/json',
                         'x-puter-events-key': call.key,
                     },
-                    body: call.body,
+                    body: call.render ? call.render() : call.body,
                 },
             );
 
@@ -319,9 +405,14 @@ export class LocalEventsInvokeTransport {
             const outcome = await this.#deployer.ensure(
                 call.appUid,
                 call.script,
+                { lane: call.lane },
             );
             if (outcome !== 'deployed')
-                return { status: null, error: `deploy: ${outcome}` };
+                return {
+                    status: null,
+                    error: `deploy: ${outcome}`,
+                    ...(outcome === 'busy' ? { deployBusy: true } : {}),
+                };
 
             const retried = await this.#dispatch(request(), call.timeoutMs);
             if (retried === null)

src/backend/core/schedules.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { isHttpError } from './http/HttpError.js';
+import {
+    SCHEDULE_LIST_LIMIT_CAP,
+    SCHEDULE_PEER_PATH,
+    schedulesUnavailable,
+} from './schedules.js';
+
+describe('schedules contract', () => {
+    it('answers an absent schedules service with a 503 events_disabled', () => {
+        const err = schedulesUnavailable();
+        expect(isHttpError(err)).toBe(true);
+        expect(err.statusCode).toBe(503);
+        expect(err.legacyCode).toBe('events_disabled');
+        // Expected on a server without schedules, so it must not page.
+        expect(err.noAlarm).toBe(true);
+    });
+
+    it('keeps the list cap and the peer path the extension relies on', () => {
+        expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
+        expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
+    });
+});

src/backend/core/schedules.ts

@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * What core asks of schedules. Nothing here is an implementation: an extension
+ * registers `services.schedules`, and a server without one answers every
+ * schedule route and `notify({ sendAt })` with `events_disabled`. Called by
+ * extensions; keep these signatures stable.
+ */
+
+import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
+import type { AppMessage } from '../services/notification/appMessage.js';
+import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
+import type { DurableSubscription } from '../stores/events/types.js';
+import type { PageResult } from '../util/pagination.js';
+import type { Actor } from './actor.js';
+import { HttpError } from './http/HttpError.js';
+
+/** Why a schedule ended. */
+export type ScheduleEndReason =
+    | 'removed'
+    | 'ended'
+    | 'app_deleted'
+    | 'uninstalled'
+    | 'permission_revoked'
+    | 'holder_gone'
+    | 'app_gone'
+    | 'orphaned'
+    | 'delivered'
+    | 'dropped';
+
+/** What happened to a set of subscription rows schedules own. */
+export type ScheduleSubscriptionChange =
+    | {
+          type: 'suspended';
+          reason: SuspendedReason;
+          /** For a reason that never lifts; defaults to that reason. */
+          endReason?: ScheduleEndReason;
+      }
+    | { type: 'resumed' }
+    | { type: 'ended'; endReason: ScheduleEndReason };
+
+/** A schedule that keeps running, one that runs once, or a delayed notification. */
+export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
+
+/**
+ * A schedule as its holder sees it. Times are epoch ms. `context` values are
+ * never returned, as for a persistent subscription: only its keys and a hash.
+ */
+export interface ScheduleView {
+    scheduleId: string;
+    kind: ScheduleViewKind;
+    appUid: string;
+    /** `null` once ended: the handler binding goes with the schedule. */
+    handlerName: string | null;
+    contextKeys: string[] | null;
+    contextHash: string | null;
+    /** Interval of an `every` schedule. */
+    every: number | null;
+    cron: string | null;
+    tz: string;
+    /** First allowed run; a one-shot's only run. */
+    startAt: number;
+    /** Last allowed run, inclusive. */
+    endsAt: number | null;
+    /** `sending` once a delayed send has started; it can't be cancelled then. */
+    state: 'active' | 'suspended' | 'sending' | 'ended';
+    suspendedReason: string | null;
+    endReason: string | null;
+    endedAt: number | null;
+    /**
+     * The slot it runs for next; `null` while suspended or sending, and once
+     * ended.
+     */
+    nextRunAt: number | null;
+    createdAt: number;
+}
+
+export interface ScheduleListRequest {
+    limit: number;
+    cursor?: string;
+    includeTotal: boolean;
+}
+
+/** The caps a holder's schedules are held to, from their plan. */
+export interface ScheduleQuota {
+    recurringPerUser: number;
+    recurringPerApp: number;
+    oneShotsPerUser: number;
+    oneShotsPerApp: number;
+    /** Worst-case runs per hour × 1000, summed over recurring rows. */
+    budgetPerUser: number;
+    budgetPerApp: number;
+}
+
+/** A caller cleared to supply `sendAt`, and when it asked for. */
+export interface DeferredSendPlan {
+    sendAt: number;
+    /** Due within 5 seconds, or past: sent now rather than stored. */
+    immediate: boolean;
+    holderUserId: number;
+    appUid: string;
+    handlerDepth: number;
+    quota: ScheduleQuota;
+}
+
+/** What a delayed app message stores until it is sent. */
+export interface DeferredAppMessage {
+    message: AppMessage;
+    /** A KV share handle naming the recipient. */
+    to?: string;
+}
+
+/**
+ * A condition on one row of another table that statements embed as `EXISTS
+ * (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
+ */
+export interface SqlGuard {
+    table: string;
+    where: string;
+    values: unknown[];
+}
+
+/** Ceiling of a schedule listing page. */
+export const SCHEDULE_LIST_LIMIT_CAP = 200;
+
+/** The signed peer route a region's schedule changes are sent to. */
+export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
+
+export const schedulesUnavailable = (): HttpError =>
+    new HttpError(503, 'Schedules are not available on this server', {
+        legacyCode: 'events_disabled',
+        // The designed answer of a server without schedules, not a fault.
+        noAlarm: true,
+    });
+
+export interface ScheduleServiceContract {
+    /**
+     * Keep schedules in step with a change to the subscription rows they own.
+     * Never throws: the subscription change has already happened.
+     */
+    onSubscriptionsChanged(
+        rows: readonly DurableSubscription[],
+        change: ScheduleSubscriptionChange,
+    ): Promise<void>;
+    createFor(
+        actor: Actor,
+        body: Record<string, unknown>,
+    ): Promise<ScheduleView>;
+    list(
+        actor: Actor,
+        request: ScheduleListRequest,
+    ): Promise<PageResult<ScheduleView>>;
+    get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
+    remove(actor: Actor, scheduleId: unknown): Promise<void>;
+    receivePeerChanges(body: unknown): Promise<{ applied: number }>;
+    /** Checks a caller supplying `sendAt`; `events_disabled` when off. */
+    prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
+    createDeferredAppMessage(
+        plan: DeferredSendPlan,
+        message: DeferredAppMessage,
+    ): Promise<{ scheduleId: string; sendAt: number }>;
+    /**
+     * Runs an events worker deploy under the schedules' deploy gate; `capped`
+     * for a schedule run's. A passthrough when schedules are off.
+     */
+    gateDeploy?(
+        script: string,
+        options: { capped: boolean },
+        deploy: () => Promise<EventsDeployOutcome>,
+    ): Promise<EventsDeployOutcome>;
+}

src/backend/drivers/notification/NotificationDriver.sendAt.test.ts

@@ -0,0 +1,493 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `notify({ sendAt })` without and with `services.schedules`, and the deferred
+ * mailbox write an extension runs inside its own transaction, fenced on a guard
+ * in a table of its own (a test table here).
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
+import { makeActor, type Actor } from '../../core/actor.js';
+import { runWithContext } from '../../core/context.js';
+import { isHttpError } from '../../core/http/HttpError.js';
+import type {
+    DeferredSendPlan,
+    ScheduleServiceContract,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import type {
+    DeferredWriteTx,
+    NotificationDriver,
+} from './NotificationDriver.js';
+
+vi.setConfig({ testTimeout: 120_000 });
+
+const BOOT_TIMEOUT_MS = 480_000;
+const TEST_TABLE = 'deferred_send_test';
+
+let server: PuterServer;
+let holder: Actor;
+let app: { id: number; uid: string };
+let saved: unknown;
+let nextId = 1;
+
+const db = () => server.clients.db;
+const driver = () =>
+    server.drivers.notifications as unknown as NotificationDriver;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+const asApp = (): Actor => makeActor({ user: holder.user, app });
+
+const notify = (fields: Record<string, unknown>): Promise<unknown> =>
+    runWithContext({ actor: asApp() }, () =>
+        driver().create({
+            object: { value: { type: 'app.message', title: 'Hi', ...fields } },
+        }),
+    );
+
+const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
+    try {
+        await run;
+    } catch (err) {
+        if (isHttpError(err)) return err.legacyCode;
+        throw err;
+    }
+    return undefined;
+};
+
+const mailbox = () =>
+    db().read(
+        'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
+            'ORDER BY `id`',
+        [holder.user!.id, app.uid],
+    );
+
+/** A row's value: JSON text on sqlite, parsed on the other engines. */
+const valueOf = (row: Record<string, unknown>) =>
+    typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
+
+/** A send claimed under `token`, as an extension's own row would be. */
+const claimed = async (token: string): Promise<number> => {
+    const id = nextId++;
+    await db().write(
+        `INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
+        [id, token],
+    );
+    return id;
+};
+
+/** The transaction an extension wraps the write in: its row, then the mailbox. */
+const txFor = (id: number, token: string): DeferredWriteTx => ({
+    guard: {
+        table: TEST_TABLE,
+        where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
+        values: [id, token],
+    },
+    async commit(entries: readonly BatchEntry[]) {
+        await db().batchWrite([
+            {
+                statement:
+                    `UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
+                    "WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
+                values: [id, token],
+            },
+            ...entries,
+        ]);
+        const [row] = await db().pread(
+            `SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
+            [id],
+        );
+        return row?.state === 'delivered' && row.token === token;
+    },
+});
+
+const write = (
+    sendKey: string,
+    tx: DeferredWriteTx,
+    over: {
+        payload?: string | null;
+        appUid?: string;
+        signal?: AbortSignal;
+    } = {},
+) =>
+    driver().deferredAppMessages.write(
+        {
+            holderUserId: holder.user!.id,
+            appUid: over.appUid ?? app.uid,
+            payload:
+                over.payload === undefined
+                    ? JSON.stringify({ message: { title: 'Later' } })
+                    : over.payload,
+            sendKey,
+            signal: over.signal ?? new AbortController().signal,
+        },
+        tx,
+    );
+
+const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
+    ({
+        onSubscriptionsChanged: vi.fn(async () => {}),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
+            sendAt: 0,
+            immediate: false,
+            holderUserId: holder.user!.id,
+            appUid: app.uid,
+            handlerDepth: 0,
+            quota: {
+                recurringPerUser: 1,
+                recurringPerApp: 1,
+                oneShotsPerUser: 1,
+                oneShotsPerApp: 1,
+                budgetPerUser: 1,
+                budgetPerApp: 1,
+            },
+            ...plan,
+        })),
+        createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
+            scheduleId: 'sched-1',
+            sendAt: p.sendAt,
+        })),
+    }) satisfies ScheduleServiceContract;
+
+beforeAll(async () => {
+    server = await setupTestServer({
+        events: { enabled: true },
+    } as IConfig);
+    const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+        requires_email_confirmation: false,
+    });
+    const row = (await server.stores.user.getById(user.id))!;
+    holder = makeActor({
+        user: {
+            id: row.id,
+            uuid: row.uuid,
+            username: row.username,
+            email: row.email ?? null,
+            email_confirmed: true,
+        } as Actor['user'],
+    });
+    const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
+    app = (await server.stores.app.create(
+        { name, title: name, index_url: `https://${name}.test/` },
+        { ownerUserId: row.id },
+    )) as { id: number; uid: string };
+    await server.services.permission.grantUserAppPermission(
+        holder,
+        app.uid,
+        EVENTS_BACKGROUND_PERMISSION,
+    );
+    await db().write(
+        `CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
+            '`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
+        [],
+    );
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
+        app.uid,
+    ]);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('notify({ sendAt }) without services.schedules', () => {
+    it('is refused events_disabled, even when due now, storing nothing', async () => {
+        setSchedules(undefined);
+        expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
+            'events_disabled',
+        );
+        expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
+            'events_disabled',
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends as before without sendAt', async () => {
+        setSchedules(undefined);
+        const sent = (await notify({ text: 'now' })) as { uid: string };
+        expect(sent.uid).toBeTruthy();
+        expect(await mailbox()).toHaveLength(1);
+    });
+});
+
+describe('notify({ sendAt }) with services.schedules', () => {
+    it('hands a later send to createDeferredAppMessage and answers its id', async () => {
+        const at = Date.now() + 3_600_000;
+        const schedules = fakeSchedules({ sendAt: at });
+        setSchedules(schedules);
+
+        await expect(notify({ sendAt: at, text: 'later' })).resolves.toEqual({
+            scheduled: { scheduleId: 'sched-1', sendAt: at },
+        });
+        expect(schedules.prepareSendAt).toHaveBeenCalledWith(
+            expect.objectContaining({ user: holder.user }),
+            at,
+        );
+        expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
+            expect.objectContaining({ sendAt: at }),
+            {
+                message: expect.objectContaining({
+                    title: 'Hi',
+                    text: 'later',
+                }),
+            },
+        );
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('sends at once when the plan says it is due', async () => {
+        const schedules = fakeSchedules({ immediate: true });
+        setSchedules(schedules);
+
+        const sent = (await notify({ sendAt: Date.now() })) as {
+            uid: string;
+        };
+        expect(sent.uid).toBeTruthy();
+        expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
+        expect(await mailbox()).toHaveLength(1);
+    });
+
+    it('passes a refusal from prepareSendAt through', async () => {
+        const schedules = fakeSchedules({});
+        schedules.prepareSendAt.mockRejectedValueOnce(
+            Object.assign(new Error('scoped'), {}),
+        );
+        setSchedules(schedules);
+        await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+describe('deferredAppMessages.write', () => {
+    it('writes one row keyed by the send, in the caller`s transaction', async () => {
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(write(key, txFor(id, token))).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: key, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('writes nothing when the guard no longer holds: another claim took it', async () => {
+        const id = await claimed(uuidv4());
+        const key = uuidv4();
+
+        await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('replaces the open row of its group, which takes the key', async () => {
+        const open = (await notify({ group: 'g', text: 'first' })) as {
+            uid: string;
+        };
+        const token = uuidv4();
+        const key = uuidv4();
+        const id = await claimed(token);
+
+        await expect(
+            write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g' },
+                }),
+            }),
+        ).resolves.toBe('delivered');
+        const rows = await mailbox();
+        expect(rows).toHaveLength(1);
+        expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
+        expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
+    });
+
+    it('drops a payload it can`t read, and a sender whose app is gone', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        await expect(
+            write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
+        ).resolves.toBe('dropped');
+        await expect(
+            write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
+        ).resolves.toBe('dropped');
+        expect(await mailbox()).toEqual([]);
+    });
+
+    it('writes nothing once the caller`s claim is lost', async () => {
+        const token = uuidv4();
+        const id = await claimed(token);
+        const lost = new AbortController();
+        lost.abort();
+
+        await expect(
+            write(uuidv4(), txFor(id, token), { signal: lost.signal }),
+        ).rejects.toThrow();
+        expect(await mailbox()).toEqual([]);
+    });
+});
+
+/**
+ * A second claim taking a send over while the first one's transaction is open.
+ * Needs two writers at once with row locks; sqlite serialises writers, so this
+ * runs on postgres only.
+ */
+describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
+    'a deferred write racing another writer',
+    () => {
+        /**
+         * Hold the first statement matching `match` on its connection, before
+         * or after it runs, until released.
+         */
+        const holdStatement = (match: RegExp, when: 'before' | 'after') => {
+            const client = db() as unknown as {
+                query: (
+                    target: unknown,
+                    sql: string,
+                    params?: unknown[],
+                ) => Promise<unknown>;
+            };
+            const original = client.query.bind(client);
+            let reached!: () => void;
+            let release!: () => void;
+            const atBarrier = new Promise<void>((r) => (reached = r));
+            const released = new Promise<void>((r) => (release = r));
+            let held = false;
+            vi.spyOn(client as never, 'query' as never).mockImplementation(
+                (async (target: unknown, sql: string, params?: unknown[]) => {
+                    if (held || !match.test(sql))
+                        return original(target, sql, params);
+                    held = true;
+                    if (when === 'before') {
+                        reached();
+                        await released;
+                        return original(target, sql, params);
+                    }
+                    const result = await original(target, sql, params);
+                    reached();
+                    await released;
+                    return result;
+                }) as never,
+            );
+            return { atBarrier, release };
+        };
+
+        const DELIVERED = /SET `state` = 'delivered'/;
+        const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
+
+        /** Whether `promise` is still pending after `ms`. */
+        const stillPending = async (promise: Promise<unknown>, ms: number) => {
+            let settled = false;
+            const watched = (async () => {
+                try {
+                    await promise;
+                } finally {
+                    settled = true;
+                }
+            })();
+            await new Promise((r) => setTimeout(r, ms));
+            void watched;
+            return !settled;
+        };
+
+        it('a transaction that opened before a takeover writes nothing', async () => {
+            const stale = uuidv4();
+            const id = await claimed(stale);
+            const key = uuidv4();
+            const gate = holdStatement(DELIVERED, 'before');
+            const first = write(key, txFor(id, stale));
+            await gate.atBarrier;
+
+            await db().write(
+                `UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
+                [uuidv4(), id],
+            );
+            gate.release();
+            await expect(first).resolves.toBe('lost');
+            expect(
+                await server.stores.notification.getBySendKey(key),
+            ).toBeNull();
+            expect(await mailbox()).toEqual([]);
+        });
+
+        it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
+            const open = (await notify({ group: 'g', text: 'first' })) as {
+                uid: string;
+            };
+            const token = uuidv4();
+            const id = await claimed(token);
+            const key = uuidv4();
+            const gate = holdStatement(REPLACE, 'after');
+            const deferred = write(key, txFor(id, token), {
+                payload: JSON.stringify({
+                    message: { title: 'Later', group: 'g', text: 'later' },
+                }),
+            });
+            await gate.atBarrier;
+
+            const immediate = notify({ group: 'g', text: 'newer' });
+            // Past the group lock's wait, then blocked on the row.
+            expect(await stillPending(immediate, 2_500)).toBe(true);
+            gate.release();
+            await expect(deferred).resolves.toBe('delivered');
+            await immediate;
+
+            const rows = await mailbox();
+            expect(rows).toHaveLength(1);
+            expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
+            expect(valueOf(rows[0]).text).toBe('newer');
+        });
+    },
+);

src/backend/drivers/notification/NotificationDriver.ts

@@ -51,6 +51,14 @@ import {
     resolveNotificationWrite,
     type NotificationTypeName,
 } from '../../services/notification/notificationTypes.js';
+import {
+    schedulesUnavailable,
+    type DeferredAppMessage,
+    type DeferredSendPlan,
+    type ScheduleServiceContract,
+    type SqlGuard,
+} from '../../core/schedules.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { withRedisLock } from '../../util/redisLock.js';
 import { PuterDriver } from '../types.js';
 import type { Actor } from '../../core/actor.js';
@@ -67,6 +75,44 @@ const GROUP_LOCK_RETRY_MS = 25;
 
 type AppMessageWindow = 'minute' | 'day';
 
+export type { SqlGuard };
+
+/** What a delayed app message's fire-time write needs from its row. */
+export interface DeferredWriteInput {
+    holderUserId: number;
+    appUid: string;
+    /** The stored {@link DeferredAppMessage}, as JSON text. */
+    payload: string | null;
+    /** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
+    sendKey: string;
+    /** Aborted when the caller's claim is lost. */
+    signal: AbortSignal;
+}
+
+/**
+ * The caller's transaction the mailbox write commits in. Atomicity is the
+ * caller's to keep; the type can't enforce it.
+ */
+export interface DeferredWriteTx {
+    /**
+     * What every mailbox statement is fenced on: the caller's row marked
+     * delivered by this claim.
+     */
+    guard: SqlGuard;
+    /**
+     * Must be the whole delivery in one step: the caller's guarded transition
+     * of its row to delivered and `entries`, in that order, in one
+     * `batchWrite`, then a read of the row on the primary. True only when that
+     * read shows this claim holding the row as delivered. Splitting the
+     * transition from `entries` lets one slot write two rows.
+     */
+    commit(entries: readonly BatchEntry[]): Promise<boolean>;
+}
+
+/** How a delayed send's write stood when it stopped. */
+export type DeferredWriteOutcome =
+    'delivered' | 'dropped' | 'lost' | 'senderSuspended';
+
 /** The mailbox slice a call named, in the terms the query takes it. */
 interface MailboxScope {
     audiences: readonly string[];
@@ -124,6 +170,21 @@ const parseNotifyTarget = (to: unknown): string => {
     return target.handle;
 };
 
+/** A stored delayed message, or `null` for one that can't be read back. */
+const parseDelayedAppMessage = (
+    payload: string | null,
+): DeferredAppMessage | null => {
+    try {
+        const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
+        if (typeof stored?.message?.title !== 'string') return null;
+        if (stored.to !== undefined && typeof stored.to !== 'string')
+            return null;
+        return stored;
+    } catch {
+        return null;
+    }
+};
+
 /** Whether a row belongs to the slice, as the same SQL scope selected it. */
 const inScope = (
     row: Record<string, unknown>,
@@ -354,7 +415,7 @@ export class NotificationDriver extends PuterDriver {
         object: Record<string, unknown>,
         value: Record<string, unknown>,
     ): Promise<unknown> {
-        const { type, to, ...fields } = value;
+        const { type, to, sendAt, ...fields } = value;
         if (type !== APP_MESSAGE_TYPE) {
             throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
         }
@@ -370,6 +431,12 @@ export class NotificationDriver extends PuterDriver {
         const handle = to === undefined ? undefined : parseNotifyTarget(to);
         const message = parseAppMessage(fields);
         assertValueFits(value);
+        const timing =
+            sendAt === undefined || sendAt === null
+                ? null
+                : await this.#schedules().prepareSendAt(actor, sendAt);
+        if (timing && !timing.immediate)
+            return this.#scheduleAppMessage(actor, message, handle, timing);
 
         if (handle === undefined) {
             const uid = await this.#sendAppMessage(
@@ -410,12 +477,23 @@ export class NotificationDriver extends PuterDriver {
         appUid: string,
         message: AppMessage,
     ): Promise<string> {
-        const send = () =>
-            this.#replaceOrWriteAppMessage(recipientId, appUid, message);
+        return this.#oneSendPerGroup(recipientId, appUid, message, () =>
+            this.#replaceOrWriteAppMessage(recipientId, appUid, message),
+        );
+    }
+
+    /**
+     * One send per group at a time, so a burst into an empty group writes one
+     * row that the rest replace. A lock that is busy past the wait or
+     * unreachable still sends: the race costs a second row and its day slot.
+     */
+    #oneSendPerGroup<T>(
+        recipientId: number,
+        appUid: string,
+        message: AppMessage,
+        send: () => Promise<T>,
+    ): Promise<T> {
         if (!message.group) return send();
-        // One send per group at a time, so a burst into an empty group writes
-        // one row that the rest replace. A lock that is busy past the wait or
-        // unreachable still sends: the race costs a second row and its day slot.
         return withRedisLock(
             this.clients.redis,
             `notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
@@ -430,6 +508,163 @@ export class NotificationDriver extends PuterDriver {
         );
     }
 
+    // -- Delayed sends -----------------------------------------------
+
+    #schedules(): ScheduleServiceContract {
+        const schedules = this.services.schedules;
+        if (!schedules) throw schedulesUnavailable();
+        return schedules;
+    }
+
+    /**
+     * Store an app's message to go out at `sendAt`. A recipient named through a
+     * share is checked now without spending its budget; every limit is spent
+     * when it goes, and the share is checked again then.
+     */
+    async #scheduleAppMessage(
+        actor: UserActor,
+        message: AppMessage,
+        handle: string | undefined,
+        plan: DeferredSendPlan,
+    ): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
+        if (handle !== undefined)
+            await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
+                spendBudget: false,
+            });
+        return {
+            scheduled: await this.#schedules().createDeferredAppMessage(
+                plan,
+                handle === undefined ? { message } : { message, to: handle },
+            ),
+        };
+    }
+
+    /**
+     * For extensions. A field, not a method, so `/drivers/call` can't reach it.
+     */
+    readonly deferredAppMessages = {
+        write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
+            this.#writeDeferredAppMessage(input, tx),
+    };
+
+    /**
+     * Check the sender and recipient as they stand now, spend the recipient's
+     * limits, and write the mailbox row through `tx`, fenced on its guard.
+     * Refusals are the same `HttpError`s an immediate send throws.
+     */
+    async #writeDeferredAppMessage(
+        input: DeferredWriteInput,
+        tx: DeferredWriteTx,
+    ): Promise<DeferredWriteOutcome> {
+        const { holderUserId, sendKey, signal } = input;
+        const sender = await this.services.events.resolveScheduleActor(
+            holderUserId,
+            input.appUid,
+        );
+        if (!sender) {
+            // A lagging replica can miss an app the primary has: retried.
+            const there =
+                (await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
+                (await this.stores.user.getById(holderUserId)) !== null;
+            if (there) throw new Error('the sender could not be resolved');
+            return 'dropped';
+        }
+        if (sender.user?.suspended) return 'senderSuspended';
+        // Uninstalled since, and the cancel at uninstall missed it.
+        const appId = sender.effectiveApp?.id;
+        if (
+            typeof appId !== 'number' ||
+            !(await this.stores.permission.hasAnyUserAppPerm(
+                holderUserId,
+                appId,
+            ))
+        )
+            return 'dropped';
+        const stored = parseDelayedAppMessage(input.payload);
+        if (!stored) return 'dropped';
+        const { message } = stored;
+
+        let recipientId = holderUserId;
+        let appUid = input.appUid;
+        if (stored.to !== undefined) {
+            const recipient =
+                await this.services.kvShare.resolveNotifyRecipient(
+                    sender,
+                    stored.to,
+                );
+            recipientId = recipient.recipientUserId;
+            appUid = recipient.appUid;
+        }
+
+        return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
+            // Primary, as for an immediate send.
+            const open = message.group
+                ? await this.stores.notification.getUnseenByGroup(
+                      recipientId,
+                      appUid,
+                      message.group,
+                      { primary: true },
+                  )
+                : null;
+            await this.#spendAppMessageLimits(
+                recipientId,
+                appUid,
+                open ? ['minute'] : ['minute', 'day'],
+            );
+            message.app = appMessageSender(
+                appUid,
+                await this.stores.app.getByUid(appUid),
+            );
+            // Another attempt may hold the lease now; hand the claim back.
+            if (signal.aborted) throw new Error('delayed send lost its lease');
+
+            const registered = resolveNotificationWrite(
+                APP_MESSAGE_TYPE,
+                appUid,
+            );
+            const delivered = await tx.commit(
+                this.stores.notification.sendEntries(
+                    {
+                        sendKey,
+                        userId: recipientId,
+                        value: { ...message, type: registered.type },
+                        type: registered.type,
+                        audience: registered.audience,
+                        appUid,
+                        group: message.group ?? null,
+                        replaceUid: open ? String(open.uid) : null,
+                        at: Date.now(),
+                    },
+                    tx.guard,
+                ),
+            );
+            if (!delivered) return 'lost';
+            if (open)
+                await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
+            return 'delivered';
+        });
+    }
+
+    /**
+     * A replacement that found its row shown since the lookup wrote a new one,
+     * which costs a day slot. Already written, so a full day is not a refusal.
+     */
+    async #spendDayIfFresh(
+        recipientId: number,
+        appUid: string,
+        open: Record<string, unknown>,
+        sendKey: string,
+    ): Promise<void> {
+        try {
+            const written =
+                await this.stores.notification.getBySendKey(sendKey);
+            if (written && written.uid !== open.uid)
+                await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
+        } catch {
+            // The row is written; a slot not spent is the only loss.
+        }
+    }
+
     async #replaceOrWriteAppMessage(
         recipientId: number,
         appUid: string,

src/backend/server.test.ts

@@ -36,6 +36,10 @@ import { Controller, Get } from './core/http/decorators.ts';
 import { HttpError } from './core/http/HttpError.ts';
 import { extension, extensionStore } from './extensions.ts';
 import { PuterServer } from './server.ts';
+import {
+    JobScheduler,
+    type JobDefinition,
+} from './services/jobs/JobScheduler.ts';
 import {
     allocateEphemeralPort,
     createTestUser,
@@ -1539,6 +1543,69 @@ describe('PuterServer lifecycle hooks', () => {
     });
 });
 
+describe('PuterServer shutdown during prepare', () => {
+    it('waits for prepare hooks the drain already started before tearing down', async () => {
+        const port = await allocateEphemeralPort();
+        const server = await setupTestServer({ port } as unknown as IConfig, {
+            listen: true,
+        });
+        const order: string[] = [];
+        (server.services as unknown as Record<string, unknown>).slowPrepare = {
+            onServerPrepareShutdown: async () => {
+                order.push('prepare:start');
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                order.push('prepare:done');
+            },
+            onServerShutdown: () => {
+                order.push('shutdown');
+            },
+        };
+
+        await server.prepareShutdown({ drainMs: 0 });
+        // The listener's close callback starts the prepare hooks.
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await server.shutdown();
+
+        expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
+    });
+});
+
+/**
+ * An extension may take over `services.jobs` (a fleet-wide scheduler). Core
+ * services register their jobs on it at start, so it must keep the built-in's
+ * first place in the layer.
+ */
+describe('PuterServer jobs replacement', () => {
+    it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
+        const registered: string[] = [];
+        class ReplacementJobs extends JobScheduler {
+            override register(name: string, definition: JobDefinition) {
+                registered.push(name);
+                super.register(name, definition);
+            }
+        }
+        const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
+        extension.registerService('jobs', ReplacementJobs as never);
+        let server: PuterServer | undefined;
+        try {
+            server = await setupTestServer();
+            const { jobs } = server.services;
+            expect(jobs).toBeInstanceOf(ReplacementJobs);
+            expect(Object.keys(server.services)[0]).toBe('jobs');
+            expect(extension.import('service').jobs).toBeInstanceOf(
+                ReplacementJobs,
+            );
+            // Only the replacement gets lifecycle hooks.
+            expect(started.mock.contexts).toEqual([jobs]);
+            expect(registered).toContain('auth.sessions.purge');
+        } finally {
+            delete (extensionStore.services as Record<string, unknown>).jobs;
+            started.mockRestore();
+            await server?.shutdown();
+        }
+    });
+});
+
 /**
  * Each layer's `onServerShutdown` has to run while the layers beneath it (the
  * ones it writes through) are still up, so teardown goes top-down: drivers,

src/backend/server.ts

@@ -1765,16 +1765,19 @@ export class PuterServer {
         }
     }
 
-    #prepareShutdownHooksRan = false;
+    #prepareShutdownHooks: Promise<void> | null = null;
 
     /**
      * Run every layer's `onServerPrepareShutdown` exactly once, whichever of
-     * `prepareShutdown()` / `shutdown()` gets there first.
+     * `prepareShutdown()` / `shutdown()` gets there first; the other waits for
+     * that run to finish.
      */
-    async #runPrepareShutdownHooks() {
-        if (this.#prepareShutdownHooksRan) return;
-        this.#prepareShutdownHooksRan = true;
-        await this.#runHook('bottomUp', 'onServerPrepareShutdown');
+    #runPrepareShutdownHooks(): Promise<void> {
+        this.#prepareShutdownHooks ??= this.#runHook(
+            'bottomUp',
+            'onServerPrepareShutdown',
+        );
+        return this.#prepareShutdownHooks;
     }
 
     async prepareShutdown(info: ServerDrainInfo = {}) {

src/backend/services/auth/AuthService.test.ts

@@ -24,6 +24,7 @@ import { makeActor, type Actor } from '../../core/actor.js';
 import { PuterServer } from '../../server.js';
 import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
 import { setupTestServer } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import { generateDefaultFsentries } from '../../util/userProvisioning.js';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS } from '../permission/consts.js';
@@ -4048,7 +4049,6 @@ describe('AuthService (integration)', () => {
     });
 
     describe('purgeEndedSessions', () => {
-        const GATE_KEY = 'sessions:purge:gate';
         const DAY = 24 * 60 * 60;
 
         const makeSession = async (
@@ -4077,8 +4077,15 @@ describe('AuthService (integration)', () => {
             return rows.length > 0;
         };
 
+        const idOf = async (uuid: string) => {
+            const [row] = await server.clients.db.read(
+                'SELECT `id` FROM `sessions` WHERE `uuid` = ?',
+                [uuid],
+            );
+            return Number(row.id);
+        };
+
         it('deletes rows ended past their retention and keeps the rest', async () => {
-            await server.clients.redis.del(GATE_KEY);
             const user = await makeUser();
             const id = user.id;
             const oldRevoked = await makeSession(id, 'asset', 'revoked_at', 31);
@@ -4100,7 +4107,7 @@ describe('AuthService (integration)', () => {
             const live = await makeSession(id, 'web');
 
             expect(
-                await authService.purgeEndedSessions(),
+                await authService.purgeEndedSessions(testJobContext()),
             ).toBeGreaterThanOrEqual(3);
 
             expect(await exists(oldRevoked)).toBe(false);
@@ -4112,34 +4119,84 @@ describe('AuthService (integration)', () => {
             expect(await exists(live)).toBe(true);
         });
 
-        it('skips while another run holds the gate', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            await server.clients.redis.set(GATE_KEY, 'other-node');
+        it('starts from the saved cursor and clears it once the walk is done', async () => {
             const user = await makeUser();
-            const oldRevoked = await makeSession(
-                user.id,
-                'asset',
-                'revoked_at',
-                31,
-            );
+            const below = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const above = await makeSession(user.id, 'asset', 'revoked_at', 31);
+            const ctx = testJobContext({ afterId: await idOf(below) });
 
-            expect(await authService.purgeEndedSessions()).toBeNull();
-            expect(await exists(oldRevoked)).toBe(true);
-            await server.clients.redis.del(GATE_KEY);
+            expect(
+                await authService.purgeEndedSessions(ctx),
+            ).toBeGreaterThanOrEqual(1);
+            expect(await exists(below)).toBe(true);
+            expect(await exists(above)).toBe(false);
+            expect(ctx.saved).toEqual([null]);
         });
 
-        it('frees the gate when the walk fails', async () => {
-            await server.clients.redis.del(GATE_KEY);
-            const spy = vi
-                .spyOn(server.clients.db, 'read')
-                .mockRejectedValueOnce(new Error('db down'));
+        it('clears its own checkpoints once a walk completes, so the next run starts at zero', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 11,
+            );
+            const ranges: number[] = [];
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    return 0;
+                });
             try {
-                await expect(authService.purgeEndedSessions()).rejects.toThrow(
-                    'db down',
+                const first = testJobContext();
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(11);
+                // Checkpointed at ten ranges, then cleared at the end.
+                expect(first.saved).toEqual([{ afterId: SPAN * 10 }, null]);
+
+                ranges.length = 0;
+                await authService.purgeEndedSessions(
+                    testJobContext(first.saved.at(-1)),
                 );
-                expect(await server.clients.redis.get(GATE_KEY)).toBeNull();
+                expect(ranges[0]).toBe(0);
+                expect(ranges).toHaveLength(11);
             } finally {
-                spy.mockRestore();
+                purge.mockRestore();
+                vi.restoreAllMocks();
+            }
+        });
+
+        it('stops between ranges once aborted, and the next run resumes at the saved cursor', async () => {
+            const SPAN = 5000;
+            vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
+                SPAN * 30,
+            );
+            const ranges: number[] = [];
+            const first = testJobContext();
+            const purge = vi
+                .spyOn(server.stores.session, 'purgeEndedInRange')
+                .mockImplementation(async (afterId: number) => {
+                    ranges.push(afterId);
+                    // Aborted mid-walk, as a lost lease or a shutdown would.
+                    if (ranges.length === 12) first.abort();
+                    return 0;
+                });
+            try {
+                await authService.purgeEndedSessions(first);
+                expect(ranges).toHaveLength(12);
+                // Saved every 10 ranges, and where it stopped.
+                expect(first.saved).toEqual([
+                    { afterId: SPAN * 10 },
+                    { afterId: SPAN * 12 },
+                ]);
+
+                ranges.length = 0;
+                const next = testJobContext(first.saved.at(-1));
+                await authService.purgeEndedSessions(next);
+                expect(ranges[0]).toBe(SPAN * 12);
+                expect(ranges.at(-1)).toBe(SPAN * 29);
+                expect(next.saved.at(-1)).toBeNull();
+            } finally {
+                purge.mockRestore();
+                vi.restoreAllMocks();
             }
         });
     });

src/backend/services/auth/AuthService.ts

@@ -46,6 +46,7 @@ import type { puterServices } from '../index';
 import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
 import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
 import type { GrantMeta } from '../permission/PermissionService.js';
+import { savedAfterId, type JobRunContext } from '../jobs/JobScheduler.js';
 import { PuterService } from '../types';
 import type {
     AccessTokenPayload,
@@ -64,13 +65,14 @@ const MAX_ORIGIN_UID_GENERATIONS = 8;
 // rows double as sign-in history, so they stay longer.
 const SESSION_RETENTION_SECONDS = 30 * 24 * 60 * 60;
 const WEB_SESSION_RETENTION_SECONDS = 365 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per week.
-const SESSION_PURGE_CHECK_MS = 60 * 60 * 1000;
-const SESSION_PURGE_GATE_KEY = 'sessions:purge:gate';
-const SESSION_PURGE_GATE_SECONDS = 7 * 24 * 60 * 60;
+const SESSION_PURGE_JOB = 'auth.sessions.purge';
+const SESSION_PURGE_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
+const SESSION_PURGE_LEASE_MS = 30 * 60 * 1000;
 // Ids per DELETE, with a pause between, so replicas keep up.
 const SESSION_PURGE_SPAN = 5000;
 const SESSION_PURGE_PAUSE_MS = 50;
+/** Ranges walked between saves of the cursor. */
+const SESSION_PURGE_SAVE_EVERY = 10;
 
 const nowSeconds = (): number => Math.floor(Date.now() / 1000);
 
@@ -138,11 +140,17 @@ export interface AuthResult {
 export class AuthService extends PuterService {
     declare protected services: LayerInstances<typeof puterServices>;
 
-    #sessionPurge: ReturnType<typeof setInterval> | null = null;
-    #stopping = false;
-
     override onServerStart(): void {
-        this.#armSessionPurge();
+        this.services.jobs.register(SESSION_PURGE_JOB, {
+            every: SESSION_PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: SESSION_PURGE_LEASE_MS,
+            run: async (ctx) => {
+                const purged = await this.purgeEndedSessions(ctx);
+                console.log(`[auth] purged ${purged} ended sessions`);
+            },
+        });
 
         // Users implicitly hold read access to their own email — needed for
         // any permission-gated path that asks for `user:<uuid>:email:read`
@@ -164,85 +172,54 @@ export class AuthService extends PuterService {
         });
     }
 
-    override onServerPrepareShutdown(): void {
-        this.#stopping = true;
-        if (this.#sessionPurge) clearInterval(this.#sessionPurge);
-        this.#sessionPurge = null;
-    }
-
     // -- Session purge -----------------------------------------------
 
     /**
      * Delete sessions revoked or expired longer ago than their retention,
-     * walking the table by id. Returns the rows deleted, or null when another
-     * node already holds this week's gate.
+     * walking the table by id from the cursor an interrupted run saved. Returns
+     * the rows deleted; stops early once `signal` aborts.
      */
-    async purgeEndedSessions(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            SESSION_PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            SESSION_PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeEndedSessions({
+        signal,
+        state,
+        saveState,
+    }: JobRunContext): Promise<number> {
         const now = nowSeconds();
         const cutoffs = {
             cutoff: now - SESSION_RETENTION_SECONDS,
             webCutoff: now - WEB_SESSION_RETENTION_SECONDS,
         };
+        const maxId = await this.stores.session.maxId();
+        let afterId = savedAfterId(state);
         let purged = 0;
-        let finished = false;
-        try {
-            const maxId = await this.stores.session.maxId();
-            for (
-                let afterId = 0;
-                afterId < maxId;
-                afterId += SESSION_PURGE_SPAN
-            ) {
-                if (this.#stopping) return purged;
-                purged += await this.stores.session.purgeEndedInRange(
-                    afterId,
-                    SESSION_PURGE_SPAN,
-                    cutoffs,
-                );
-                await new Promise((resolve) =>
-                    setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A walk cut short frees the gate, so another node restarts it on
-            // its next check instead of next week.
-            if (!finished) {
+        let unsaved = 0;
+        while (afterId < maxId) {
+            if (signal.aborted) {
+                // Kept for the next run if this one still holds the job.
                 try {
-                    await this.clients.redis.del(SESSION_PURGE_GATE_KEY);
+                    await saveState({ afterId });
                 } catch {
-                    // The gate expires on its own.
+                    // Lost the job; the next run redoes these ranges.
                 }
+                return purged;
             }
-        }
-    }
-
-    #armSessionPurge(): void {
-        const timer = setInterval(() => {
-            void this.#runSessionPurge();
-        }, SESSION_PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#sessionPurge = timer;
-    }
-
-    async #runSessionPurge(): Promise<void> {
-        try {
-            const purged = await this.purgeEndedSessions();
-            if (purged !== null) {
-                console.log(`[auth] purged ${purged} ended sessions`);
+            purged += await this.stores.session.purgeEndedInRange(
+                afterId,
+                SESSION_PURGE_SPAN,
+                cutoffs,
+            );
+            afterId += SESSION_PURGE_SPAN;
+            if (++unsaved >= SESSION_PURGE_SAVE_EVERY) {
+                await saveState({ afterId });
+                unsaved = 0;
             }
-        } catch (err) {
-            console.warn('[auth] session purge failed', err);
+            await new Promise((resolve) =>
+                setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
+            );
         }
+        // Clears any checkpoint, this run's included, so next week starts at 0.
+        await saveState(null);
+        return purged;
     }
 
     // -- Public API --------------------------------------------------

src/backend/services/events/EventsService.test.ts

@@ -293,12 +293,14 @@ const buildService = (
     metered: MeteredLine[];
     forwarded: ForwardedCall[];
     eventBus: { on: ReturnType<typeof vi.fn>; emit: ReturnType<typeof vi.fn> };
+    jobs: { register: ReturnType<typeof vi.fn> };
 } => {
     const outbox: Array<{ socket?: string; envelope: DeliveryEnvelope }> = [];
     const counted: DeliveryEnvelope[] = [];
     const lines: MeteredLine[] = [];
     const forwardedCalls: ForwardedCall[] = [];
     const bus = { on: vi.fn(), emit: vi.fn() };
+    const jobs = { register: vi.fn() };
     const built = new EventsService(
         config,
         {
@@ -315,6 +317,7 @@ const buildService = (
             kvShareHandle: kvShareHandleStore,
         } as never,
         {
+            jobs,
             eventForward: {
                 // A deployment with no peers has nowhere to forward to, which
                 // is what every test here is.
@@ -376,6 +379,7 @@ const buildService = (
         metered: lines,
         forwarded: forwardedCalls,
         eventBus: bus,
+        jobs,
     };
 };
 
@@ -564,6 +568,52 @@ describe('the feature switch', () => {
         );
     });
 
+    it('hands its table sweeps to the job scheduler and arms only the pending sweep itself', () => {
+        const intervals = vi.spyOn(globalThis, 'setInterval');
+        const timeouts = vi.spyOn(globalThis, 'setTimeout');
+        let jobs: { register: ReturnType<typeof vi.fn> };
+        try {
+            ({ jobs } = buildService({ events: { enabled: true } } as IConfig));
+            // The pending sweep is the region's queue, so every node runs it.
+            expect(intervals.mock.calls.map(([, ms]) => ms)).toEqual([10_000]);
+            expect(timeouts).not.toHaveBeenCalled();
+        } finally {
+            intervals.mockRestore();
+            timeouts.mockRestore();
+        }
+        expect(
+            jobs.register.mock.calls.map(([name, job]) => ({
+                name,
+                every: job.every,
+                scope: job.scope,
+                nearPrimary: job.nearPrimary ?? false,
+                leaseMs: job.leaseMs,
+            })),
+        ).toEqual([
+            {
+                name: 'events.subscriptions.expirySweep',
+                every: 60 * 60_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: 15 * 60_000,
+            },
+            {
+                name: 'events.subscriptions.creditSweep',
+                every: 15 * 60_000,
+                scope: 'fleet',
+                nearPrimary: false,
+                leaseMs: 10 * 60_000,
+            },
+        ]);
+    });
+
+    it('registers no job when off', () => {
+        const { jobs } = buildService({
+            events: { enabled: false },
+        } as IConfig);
+        expect(jobs.register).not.toHaveBeenCalled();
+    });
+
     it('spends nothing at all on a dispatch when off', async () => {
         const { service: off } = buildService({
             events: { enabled: false },

src/backend/services/events/appDeletion.integration.test.ts

@@ -29,6 +29,7 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
 import { EVENTS_WORKER_SESSION_NAME } from './workerRuntime.js';
@@ -327,7 +328,9 @@ describe('an app deleted while it has events state', () => {
         await env.server.stores.app.delete((row as { id: number }).id);
 
         const revoked =
-            await env.server.services.events.sweepStrandedWorkerSessions();
+            await env.server.services.events.sweepStrandedWorkerSessions(
+                testJobContext(),
+            );
         expect(revoked).toBeGreaterThanOrEqual(1);
 
         // `authenticate()` can no longer resolve the token at all — the app
@@ -366,9 +369,60 @@ describe('an app deleted while it has events state', () => {
             EVENTS_WORKER_SESSION_NAME,
         );
 
-        await env.server.services.events.sweepStrandedWorkerSessions();
+        await env.server.services.events.sweepStrandedWorkerSessions(
+            testJobContext(),
+        );
 
         const reauth = await env.server.services.auth.authenticate(token);
         expect(reauth).not.toHaveProperty('reauth');
     });
+
+    it('reaches a stranded session past one run`s two pages on the next run, then wraps', async () => {
+        const db = env.server.clients.db;
+        const now = Math.floor(Date.now() / 1000);
+        const meta = JSON.stringify({
+            worker_name: EVENTS_WORKER_SESSION_NAME,
+        });
+        const workerSession = (appUid: string) => ({
+            statement:
+                'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`, `kind`, `app_uid`) ' +
+                "VALUES (?, ?, ?, ?, ?, 'worker', ?)",
+            values: [uuidv4(), userId, meta, now, now, appUid],
+        });
+        // A run reads two pages of 500; these fill them, all still standing.
+        const live = Array.from(
+            { length: 1000 },
+            (_, i) => `app-live-${uuidv4().slice(0, 8)}-${i}`,
+        );
+        await db.batchWrite(
+            live.map((uid) => ({
+                statement:
+                    'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
+                values: [uid, uid, uid, `https://${uid}.example/`, userId],
+            })),
+        );
+        await db.batchWrite(live.map(workerSession));
+        const stranded = workerSession(`app-gone-${uuidv4()}`);
+        await db.batchWrite([stranded]);
+        const strandedRevoked = async () => {
+            const [row] = await db.pread(
+                'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
+                [stranded.values[0]],
+            );
+            return row.revoked_at !== null;
+        };
+
+        const first = testJobContext();
+        await env.server.services.events.sweepStrandedWorkerSessions(first);
+        expect(await strandedRevoked()).toBe(false);
+        expect(first.saved.at(-1)).toMatchObject({
+            afterId: expect.any(Number),
+        });
+
+        const second = testJobContext(first.saved.at(-1));
+        await env.server.services.events.sweepStrandedWorkerSessions(second);
+        expect(await strandedRevoked()).toBe(true);
+        // The end of the table wraps the cursor back to the start.
+        expect(second.saved.at(-1)).toBeNull();
+    });
 });

src/backend/services/events/durable.integration.test.ts

@@ -872,6 +872,71 @@ describe('what a write costs once a durable row exists', () => {
 
         expect(deliveryOf(path)).toBeUndefined();
     });
+
+    it('publishes the expiry reap`s bump, so a region still caching the row drops it', async () => {
+        await clearRows();
+        const created = await subscribe(env.users.user.token, {
+            expiresAt: Math.floor(Date.now() / 1000) + 3600,
+        });
+        const subId = created.body.subId as string;
+        const stale =
+            await env.server.stores.durableSubscription.getBySubId(subId);
+        await env.server.clients.db.write(
+            `UPDATE \`${TABLE}\` SET \`expires_at\` = ? WHERE \`sub_id\` = ?`,
+            [Math.floor(Date.now() / 1000) - 1, subId],
+        );
+
+        const heard: Array<Record<string, unknown>> = [];
+        const listener = (_key: string, data: unknown) => {
+            heard.push(data as Record<string, unknown>);
+        };
+        const announced = vi.spyOn(
+            env.server.services.eventForward,
+            'announceGeneration',
+        );
+        env.server.clients.event.on(
+            'outer.pubsub.events.generationBumped',
+            listener as never,
+        );
+        try {
+            await expect(events().sweepExpired()).resolves.toBe(1);
+            // Over the addressed peer channel as well as the bus.
+            expect(announced).toHaveBeenCalledWith(
+                expect.objectContaining({ userId }),
+                true,
+                'subscription',
+            );
+        } finally {
+            env.server.clients.event.off(
+                'outer.pubsub.events.generationBumped',
+                listener as never,
+            );
+            announced.mockRestore();
+        }
+        const published = heard.find(
+            (bump) => bump.userId === userId && bump.durable === true,
+        );
+        expect(published).toBeDefined();
+
+        // Another region's Redis still holds the row; this one stands in for it.
+        await env.server.stores.eventSubscription.cacheDurable([stale!]);
+        delivered.length = 0;
+        const before = `${anchor}/stale-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: before });
+        await settle(before);
+        expect(deliveryOf(before)?.subId).toBe(subId);
+
+        await env.server.clients.event.emitAndWait(
+            'outer.pubsub.events.generationBumped',
+            published!,
+            { from_outside: true },
+        );
+        delivered.length = 0;
+        const after = `${anchor}/peer-swept-${uuidv4().slice(0, 8)}.txt`;
+        await fs().touch(userId, { path: after });
+        await quiet();
+        expect(deliveryOf(after)).toBeUndefined();
+    });
 });
 
 describe('where a durable delivery is addressed', () => {

src/backend/services/events/forwarding.test.ts

@@ -79,6 +79,14 @@ vi.mock(import('@opentelemetry/api'), async (importOriginal) => ({
                     metricCalls.push({ name, value, attributes });
                 },
             }),
+            createHistogram: (name: string) => ({
+                record: (
+                    value: number,
+                    attributes: Record<string, unknown> = {},
+                ) => {
+                    metricCalls.push({ name, value, attributes });
+                },
+            }),
         }),
     },
 }));
@@ -441,6 +449,7 @@ const makeRegion = (
             ): Promise<{
                 row: DurableSubscription;
                 bump: { userId: number; generation: number };
+                reaped: [];
             }> => {
                 const row = durableRow({
                     holderUserId: input.holderUserId as number,
@@ -469,6 +478,7 @@ const makeRegion = (
                             row.ownerUserId,
                         ),
                     },
+                    reaped: [],
                 };
             },
         },

src/backend/services/events/handlers.integration.test.ts

@@ -220,7 +220,8 @@ afterAll(async () => {
 
 beforeEach(async () => {
     await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
-    for (const row of await durable().listActiveForHolder(userId, null))
+    for (const row of (await durable().listActiveForHolderPage(userId, null, 0))
+        .rows)
         await durable().remove(row);
     await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
     await env.server.stores.eventSubscription.rebuildDurable(userId, []);
@@ -777,7 +778,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(404);
         expect(refused.body.code).toBe('events_handler_not_found');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('binds an inline body whose hash is what is published', async () => {
@@ -805,7 +808,9 @@ describe('binding a subscription to a handler', () => {
 
         expect(refused.status).toBe(409);
         expect(refused.body.code).toBe('events_handler_hash_mismatch');
-        expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
+        expect(
+            (await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
+        ).toEqual([]);
     });
 
     it('still needs a name for a subscription owed to one consumer', async () => {

src/backend/services/events/kvHandleRevoke.integration.test.ts

@@ -383,7 +383,7 @@ describe('revoking a handle', () => {
         const sub = await subscribeDurable(handle);
 
         const store = env.server.stores.durableSubscription;
-        const byHolder = vi.spyOn(store, 'listActiveForHolder');
+        const byHolder = vi.spyOn(store, 'listActiveForHolderPage');
         const settle = vi.spyOn(events(), 'settleRevokedGrant');
 
         try {
@@ -400,7 +400,9 @@ describe('revoking a handle', () => {
             for (const call of byHolder.mock.calls)
                 expect(call.slice(1)).toEqual([
                     null,
-                    { includeSuspended: true },
+                    0,
+                    undefined,
+                    { includeSuspended: true, excludeSchedules: true },
                 ]);
         } finally {
             byHolder.mockRestore();
@@ -442,7 +444,9 @@ describe('revoking a handle', () => {
         const page = await shares().listKvHandles(owner.actor, {
             limit: 100,
         });
-        const deeperRow = page.items.find((one) => one.handle === deeper.handle);
+        const deeperRow = page.items.find(
+            (one) => one.handle === deeper.handle,
+        );
         expect(deeperRow?.revokedAt).toBeTypeOf('number');
         expect((await countLive()).total).toBe(before.total);
     });

src/backend/services/events/metering.test.ts

@@ -36,13 +36,17 @@ import {
     DurableQuotaRaceLost,
     type DurableSubscriptionInput,
 } from '../../stores/events/DurableSubscriptionStore.js';
-import type { DurableSubscription } from '../../stores/events/types.js';
+import {
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import {
     DEFAULT_FREE_SUBSCRIPTION,
     DEFAULT_TEMP_SUBSCRIPTION,
 } from '../metering/consts.js';
 import type { UsageInput } from '../metering/types.js';
+import { testJobContext } from '../jobs/jobsTestUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENTS_COSTS } from './costs.js';
 import { EventsService } from './EventsService.js';
@@ -62,6 +66,12 @@ let solvent: Set<number>;
 /** Set to have the next create lose the last slot after its insert. */
 let raceLost: boolean;
 let emitted: Array<{ key: string; data: unknown }>;
+/** Rows one credit-sweep page takes; null for the service's own batch. */
+let pageSize: number | null;
+/** Rows one holder page takes; null for all of them in one page. */
+let holderPageSize: number | null;
+/** What `services.schedules` hears about changed schedule rows. */
+let schedulesChanged: (rows: readonly DurableSubscription[]) => void;
 
 const anchorUid = (): string => `docs-${seq}`;
 const anchorPath = (): string => `/u${userId}/Documents`;
@@ -99,7 +109,10 @@ const actorFor = (over: Partial<Actor> = {}): Actor =>
     }) as unknown as Actor;
 
 const appActor = (appUid: string): Actor =>
-    actorFor({ app: { uid: appUid, id: 1 }, effectiveApp: { uid: appUid, id: 1 } });
+    actorFor({
+        app: { uid: appUid, id: 1 },
+        effectiveApp: { uid: appUid, id: 1 },
+    });
 
 const anchorEntry = (): FSEntry =>
     ({
@@ -124,6 +137,9 @@ beforeEach(() => {
     solvent = new Set([userId]);
     raceLost = false;
     emitted = [];
+    pageSize = null;
+    holderPageSize = null;
+    schedulesChanged = () => undefined;
 
     service = new EventsService(
         { events: { enabled: true } } as IConfig,
@@ -155,26 +171,49 @@ beforeEach(() => {
                     return {
                         row,
                         bump: { userId: input.ownerUserId, generation: 1 },
+                        reaped: [],
                     };
                 },
-                listSuspendedPage: async (reason: string, afterId: number) =>
-                    afterId === 0
-                        ? {
-                              rows: rows.filter(
-                                  (row) => row.suspendedReason === reason,
-                              ),
-                              nextId: null,
-                          }
-                        : { rows: [], nextId: null },
-                listSuspendedForHolder: async (
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedPage: async (
+                    reason: string,
+                    afterId: number,
+                    batchSize: number,
+                ) => {
+                    const size = pageSize ?? batchSize;
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId && row.suspendedReason === reason,
+                        )
+                        .slice(0, size);
+                    return {
+                        rows: page.map(({ row }) => row),
+                        nextId: page.length < size ? null : page.at(-1)!.id,
+                    };
+                },
+                // Keyset paging over `rows`, a row's id being its position + 1.
+                listSuspendedForHolderPage: async (
                     holderUserId: number,
                     reason: string,
-                ) =>
-                    rows.filter(
-                        (row) =>
-                            row.holderUserId === holderUserId &&
-                            row.suspendedReason === reason,
-                    ),
+                    afterId: number,
+                ) => {
+                    const page = rows
+                        .map((row, i) => ({ row, id: i + 1 }))
+                        .filter(
+                            ({ row, id }) =>
+                                id > afterId &&
+                                row.holderUserId === holderUserId &&
+                                row.suspendedReason === reason,
+                        );
+                    const size = holderPageSize ?? page.length;
+                    const taken = page.slice(0, size);
+                    return {
+                        rows: taken.map(({ row }) => row),
+                        nextId: page.length > size ? taken.at(-1)!.id : null,
+                    };
+                },
                 resume: async (resuming: readonly DurableSubscription[]) => {
                     for (const row of resuming) {
                         resumed.push(row.subId);
@@ -233,10 +272,7 @@ beforeEach(() => {
             // plan allows, not about who agreed to it.
             permission: { check: async () => true },
             metering: {
-                bufferIncrementUsages: (
-                    actor: Actor,
-                    usages: UsageInput[],
-                ) => {
+                bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
                     for (const usage of usages)
                         metered.push({
                             userUuid: actor.user?.uuid,
@@ -247,6 +283,11 @@ beforeEach(() => {
                     solvent.has(Number(actor.user?.id ?? -1)),
                 getActorSubscription: async () => ({ id: plan }),
             },
+            schedules: {
+                onSubscriptionsChanged: async (
+                    changed: readonly DurableSubscription[],
+                ) => schedulesChanged(changed),
+            },
         } as never,
     );
 });
@@ -258,23 +299,105 @@ describe('a suspension waiting on a balance', () => {
             suspendedReason: 'no_credit',
         });
 
-    it('comes back once the account has budget again', async () => {
+    it('comes back once the account has budget again, and every region hears it', async () => {
         const row = suspended();
         rows.push(row);
 
-        await expect(service.sweepNoCredit()).resolves.toBe(1);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(1);
         expect(resumed).toEqual([row.subId]);
         expect(row.suspendedReason).toBeNull();
+        expect(emitted).toContainEqual({
+            key: 'outer.pubsub.events.generationBumped',
+            data: { userId, generation: 1, durable: true },
+        });
     });
 
     it('stays out of service while the account still has none', async () => {
         rows.push(suspended());
         solvent.clear();
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 
+    it('reaches rows past one run`s page cap across runs, then wraps', async () => {
+        // One row per page, so a run's 200 pages stop at 200 rows.
+        pageSize = 1;
+        for (let i = 0; i < 450; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 10_000 + i;
+        // Only the last holder has a balance again.
+        solvent = new Set([10_000 + 449]);
+
+        const first = testJobContext();
+        await expect(service.sweepNoCredit(first)).resolves.toBe(0);
+        expect(first.saved.at(-1)).toEqual({ afterId: 200 });
+
+        const second = testJobContext(first.saved.at(-1));
+        await expect(service.sweepNoCredit(second)).resolves.toBe(0);
+        expect(second.saved.at(-1)).toEqual({ afterId: 400 });
+
+        const third = testJobContext(second.saved.at(-1));
+        await expect(service.sweepNoCredit(third)).resolves.toBe(1);
+        expect(resumed).toEqual([rows[449].subId]);
+        // The end of the table wraps the cursor back to the start.
+        expect(third.saved.at(-1)).toBeNull();
+    });
+
+    it('stops between pages once its signal aborts, keeping the last full page', async () => {
+        pageSize = 1;
+        for (let i = 0; i < 5; i++) rows.push(suspended());
+        for (const [i, row] of rows.entries()) row.holderUserId = 20_000 + i;
+        const ctx = testJobContext();
+        const save = ctx.saveState;
+        ctx.saveState = async (next) => {
+            await save(next);
+            if (ctx.saved.length === 2) ctx.abort();
+        };
+
+        await service.sweepNoCredit(ctx);
+        expect(ctx.saved).toEqual([{ afterId: 1 }, { afterId: 2 }]);
+    });
+
+    it('resumes no later holder page once its signal aborts inside a page`s hook, keeping the checkpoint', async () => {
+        // A full sweep page, so finishing it would move the checkpoint.
+        pageSize = 3;
+        holderPageSize = 1;
+        for (let i = 0; i < 3; i++)
+            rows.push(
+                durableRow({
+                    subId: `sched-${i}`,
+                    token: scheduleToken(`schedule-${seq}-${i}`),
+                    suspendedAt: Math.floor(Date.now() / 1000),
+                    suspendedReason: 'no_credit',
+                }),
+            );
+        const ctx = testJobContext();
+        const heard: string[][] = [];
+        schedulesChanged = (changed) => {
+            heard.push(changed.map((row) => row.subId));
+            ctx.abort();
+        };
+
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(1);
+        expect(resumed).toEqual(['sched-0']);
+        expect(heard).toEqual([['sched-0']]);
+        expect(rows[1].suspendedReason).toBe('no_credit');
+        expect(ctx.saved).toEqual([]);
+    });
+
+    it('wraps a saved cursor past the end of a shrunken table back to the start', async () => {
+        rows.push(suspended());
+        const ctx = testJobContext({ afterId: 999 });
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([null]);
+    });
+
+    it('writes no cursor while there is nothing to page through', async () => {
+        const ctx = testJobContext();
+        await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
+        expect(ctx.saved).toEqual([]);
+    });
+
     it('leaves a suspension it did not cause alone', async () => {
         rows.push(
             durableRow({
@@ -283,7 +406,7 @@ describe('a suspension waiting on a balance', () => {
             }),
         );
 
-        await expect(service.sweepNoCredit()).resolves.toBe(0);
+        await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
         expect(resumed).toEqual([]);
     });
 });
@@ -298,9 +421,10 @@ describe('what a plan lets an account subscribe to', () => {
         await subscribe(appActor(`app-${seq}`));
 
         expect(created[0].limits).toEqual({
-            perUser: EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
-                DEFAULT_FREE_SUBSCRIPTION
-            ],
+            perUser:
+                EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
+                    DEFAULT_FREE_SUBSCRIPTION
+                ],
             perApp: EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP.bySubscription[
                 DEFAULT_FREE_SUBSCRIPTION
             ],

src/backend/services/events/registry.test.ts

@@ -28,6 +28,7 @@ import {
     lookupFsSubject,
     lookupKvSubject,
     lookupPublicSubject,
+    projectScheduleTick,
     type FsDeliveryContext,
     type FsPublicSubject,
     type KvDeliveryContext,
@@ -324,3 +325,52 @@ describe('registry typing', () => {
         expect(subjects.length).toBeGreaterThan(0);
     });
 });
+
+describe('the schedule tick', () => {
+    const tick = {
+        scheduleId: '2f7c1b6e-4d1a-4c55-9a8e-0e6f3c2b1a90',
+        scheduledFor: 1_798_761_600_000,
+        missed: 0,
+        missedCapped: false,
+        attempt: 1,
+    };
+
+    it('carries exactly the fields a handler is promised', () => {
+        const event = projectScheduleTick(tick);
+        expect(Object.keys(event).sort()).toEqual([
+            'attempt',
+            'id',
+            'missed',
+            'op',
+            'scheduleId',
+            'scheduledFor',
+            'subject',
+        ]);
+        expect(event).toEqual({
+            subject: `schedule:${tick.scheduleId}`,
+            op: 'tick',
+            id: `${tick.scheduleId}:${tick.scheduledFor}`,
+            scheduleId: tick.scheduleId,
+            scheduledFor: tick.scheduledFor,
+            missed: 0,
+            attempt: 1,
+        });
+    });
+
+    it('says when `missed` stopped counting, and only then', () => {
+        const capped = projectScheduleTick({
+            ...tick,
+            missed: 1000,
+            missedCapped: true,
+        });
+        expect(capped.missedCapped).toBe(true);
+        expect(capped.missed).toBe(1000);
+        expect('missedCapped' in projectScheduleTick(tick)).toBe(false);
+    });
+
+    it('keeps the id across attempts at one slot', () => {
+        expect(projectScheduleTick({ ...tick, attempt: 3 }).id).toBe(
+            projectScheduleTick(tick).id,
+        );
+    });
+});

src/backend/services/events/registry.ts

@@ -20,6 +20,7 @@
 import type { EventKey, KvOp } from '../../clients/event/types.js';
 import type { FSEntry } from '../../stores/fs/FSEntry.js';
 import type { AclMode } from '../acl/ACLService.js';
+import { scheduleToken } from '../../stores/events/types.js';
 import type { NotificationAudience } from '../notification/notificationTypes.js';
 import { relativeTo } from './matcher.js';
 import {
@@ -359,6 +360,50 @@ const notifProject = (delivery: NotifDeliveryContext): ProjectedNotifEvent => ({
     seq: delivery.seq,
 });
 
+// -- Schedule ticks ---------------------------------------------------
+
+/**
+ * What a schedule's handler is handed as `event`. No bus event stands behind
+ * it, so it has no registry entry: the runner builds it from the slot it fires.
+ * Additive only, as for the other projections.
+ */
+export interface ScheduleEvent {
+    /** `schedule:<scheduleId>`. */
+    subject: string;
+    op: 'tick';
+    /** `${scheduleId}:${scheduledFor}`: the same across retries of one slot. */
+    id: string;
+    scheduleId: string;
+    /** The slot this run is for, in epoch ms. */
+    scheduledFor: number;
+    /** When the invocation was sent; stamped as it goes out. */
+    firedAt?: number;
+    /** Slots skipped since the last one that ran. */
+    missed: number;
+    /** Set when `missed` stopped counting at its cap. */
+    missedCapped?: true;
+    /** Attempts at this slot, this one included. */
+    attempt: number;
+}
+
+/** The {@link ScheduleEvent} of one run. Called by extensions. */
+export const projectScheduleTick = (tick: {
+    scheduleId: string;
+    scheduledFor: number;
+    missed: number;
+    missedCapped: boolean;
+    attempt: number;
+}): ScheduleEvent => ({
+    subject: scheduleToken(tick.scheduleId),
+    op: 'tick',
+    id: `${tick.scheduleId}:${tick.scheduledFor}`,
+    scheduleId: tick.scheduleId,
+    scheduledFor: tick.scheduledFor,
+    missed: tick.missed,
+    ...(tick.missedCapped ? { missedCapped: true as const } : {}),
+    attempt: tick.attempt,
+});
+
 // -- Registry ---------------------------------------------------------
 
 export const PUBLIC_SUBJECTS = [

src/backend/services/events/requestParsing.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { HttpError } from '../../core/http/HttpError.js';
+import { checkRateLimit } from '../../core/http/middleware/rateLimit.js';
+import type { RouteRateLimit } from '../../core/http/types.js';
+import {
+    HANDLER_NAME_MAX_LENGTH,
+    hashContent,
+} from '../../stores/events/EventHandlerStore.js';
+
+/** Request and response pieces the subscription and schedule verbs share. */
+
+export const badRequest = (message: string, code: string): HttpError =>
+    new HttpError(400, message, { legacyCode: code });
+
+export const parseHandlerName = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || value.length === 0)
+        throw badRequest('handlerName must be a string', 'bad_request');
+    if (value.length > HANDLER_NAME_MAX_LENGTH)
+        throw badRequest(
+            `handlerName may not exceed ${HANDLER_NAME_MAX_LENGTH} characters`,
+            'bad_request',
+        );
+    return value;
+};
+
+/** Hex digest of the inline source a subscribe claims it is binding. */
+export const parseHandlerHash = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value))
+        throw badRequest(
+            'handlerHash must be a sha-256 hex digest',
+            'bad_request',
+        );
+    return value;
+};
+
+/** Stored as JSON text; the byte cap is the store's to enforce. */
+export const parseContext = (value: unknown): string | null => {
+    if (value === undefined || value === null) return null;
+    try {
+        return JSON.stringify(value);
+    } catch {
+        throw badRequest('context must be JSON-serializable', 'bad_request');
+    }
+};
+
+const tooManyCalls = (): HttpError =>
+    new HttpError(429, 'Too many subscription changes', {
+        legacyCode: 'too_many_requests',
+    });
+
+/** Spend one call from a per-user window, refusing past it. */
+export const spendUserWindow = async (
+    limit: RouteRateLimit,
+    userId: number,
+): Promise<void> => {
+    const ok = await checkRateLimit(
+        `${limit.scope}:${userId}`,
+        limit.limit,
+        limit.window,
+    );
+    if (!ok) throw tooManyCalls();
+};
+
+/**
+ * A stored context as a listing may describe it: which keys it sets, and a hash
+ * of the whole blob. Never the values: the column is where an API key lives.
+ */
+export const projectContext = (
+    context: string | null,
+): { contextKeys: string[] | null; contextHash: string | null } => {
+    if (context === null) return { contextKeys: null, contextHash: null };
+    let keys: string[] = [];
+    try {
+        const parsed: unknown = JSON.parse(context);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+            keys = Object.keys(parsed).sort();
+    } catch {
+        // Stored as JSON by the verbs, so this cannot normally happen; an
+        // unreadable blob still reports its hash rather than failing the list.
+    }
+    return { contextKeys: keys, contextHash: hashContent(context) };
+};

src/backend/services/events/scheduleSeam.integration.test.ts

@@ -0,0 +1,344 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The subscription lifecycle hands schedules' rows, and only theirs, to
+ * `services.schedules` with the change each path made. Without it every path
+ * still runs.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type {
+    ScheduleServiceContract,
+    ScheduleSubscriptionChange,
+} from '../../core/schedules.js';
+import type { PuterServer } from '../../server.js';
+import type { DurableSubscriptionInput } from '../../stores/events/DurableSubscriptionStore.js';
+import {
+    isScheduleRow,
+    scheduleToken,
+    type DurableSubscription,
+} from '../../stores/events/types.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
+
+const BOOT_TIMEOUT_MS = 120_000;
+const APP_UID = 'app-seam';
+
+let server: PuterServer;
+let userId: number;
+let anchorUid: string;
+let anchorPath: string;
+let saved: unknown;
+
+const durable = () => server.stores.durableSubscription;
+const slot = () => server.services as unknown as { schedules?: unknown };
+/** Put `schedules` in the slot, or leave the slot empty. */
+const setSchedules = (schedules: unknown): void => {
+    if (schedules === undefined) delete slot().schedules;
+    else slot().schedules = schedules;
+};
+
+const recording = () => {
+    const calls: Array<{
+        subIds: string[];
+        change: ScheduleSubscriptionChange;
+    }> = [];
+    const schedules = {
+        onSubscriptionsChanged: vi.fn(
+            async (
+                rows: readonly DurableSubscription[],
+                change: ScheduleSubscriptionChange,
+            ) => {
+                calls.push({ subIds: rows.map((r) => r.subId), change });
+            },
+        ),
+        createFor: vi.fn(),
+        list: vi.fn(),
+        get: vi.fn(),
+        remove: vi.fn(),
+        receivePeerChanges: vi.fn(),
+        prepareSendAt: vi.fn(),
+        createDeferredAppMessage: vi.fn(),
+    } satisfies ScheduleServiceContract;
+    setSchedules(schedules);
+    return calls;
+};
+
+const plainRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> =>
+    (
+        await durable().create({
+            holderUserId: userId,
+            ownerUserId: userId,
+            appUid: APP_UID,
+            subject: `fs:${anchorPath}`,
+            token: `f#${anchorUid}`,
+            anchorUid,
+            anchorPath,
+            match: null,
+            op: null,
+            delivery: 'broadcast',
+            handlerName: 'onWrite',
+            context: null,
+            permission: 'list',
+            expiresAt: null,
+            ...over,
+        })
+    ).row;
+
+const scheduleRow = async (
+    over: Partial<DurableSubscriptionInput> = {},
+): Promise<DurableSubscription> => {
+    const uid = uuidv4();
+    return plainRow({
+        subject: scheduleToken(uid),
+        token: scheduleToken(uid),
+        anchorUid: uid,
+        anchorPath: '',
+        delivery: 'single',
+        handlerName: 'tick',
+        permission: 'events:background',
+        ...over,
+    });
+};
+
+const pendingCalls = () => {
+    const pending = server.stores.pendingDelivery;
+    return [
+        vi.spyOn(pending, 'purge'),
+        vi.spyOn(pending, 'hold'),
+        vi.spyOn(pending, 'releaseHold'),
+    ];
+};
+
+beforeAll(async () => {
+    server = await setupTestServer({ events: { enabled: true } } as IConfig);
+    const username = `seam-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+    });
+    userId = user.id;
+    anchorPath = `/${username}/seam`;
+    await server.services.fs.mkdir(userId, {
+        path: anchorPath,
+        createMissingParents: true,
+    });
+    anchorUid = (await server.stores.fsEntry.getEntryByPath(anchorPath))!.uid;
+    saved = slot().schedules;
+}, BOOT_TIMEOUT_MS);
+
+beforeEach(async () => {
+    await server.clients.db.write('DELETE FROM `event_subscriptions`', []);
+});
+
+afterEach(() => {
+    setSchedules(saved);
+    vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+describe('the schedules lifecycle hook', () => {
+    it('gets only schedule rows on a suspend, and their backlog is never touched', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const [purge, hold, releaseHold] = pendingCalls();
+
+        await server.services.events.suspendSubscriptions(
+            [schedule],
+            'no_credit',
+        );
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'no_credit' },
+            },
+        ]);
+        expect(purge).not.toHaveBeenCalled();
+        expect(hold).not.toHaveBeenCalled();
+
+        calls.length = 0;
+        await server.services.events.suspendSubscriptions([plain], 'no_credit');
+        expect(calls).toEqual([]);
+
+        calls.length = 0;
+        const suspended = (await durable().getBySubId(schedule.subId))!;
+        await server.services.events.resumeSubscriptions([suspended]);
+        expect(calls).toEqual([
+            { subIds: [schedule.subId], change: { type: 'resumed' } },
+        ]);
+        expect(releaseHold).not.toHaveBeenCalled();
+    });
+
+    it('ends schedules uninstalled when every grant to the app goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: null,
+        });
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: {
+                    type: 'suspended',
+                    reason: 'permission_revoked',
+                    endReason: 'uninstalled',
+                },
+            },
+        ]);
+    });
+
+    it('reads no schedule rows to settle a specific grant, and still settles them when consent goes', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        const plain = await plainRow();
+        const page = vi.spyOn(durable(), 'listActiveForHolderPage');
+        const readSubIds = async () =>
+            (await Promise.all(page.mock.results.map((r) => r.value))).flatMap(
+                (p) => p.rows.map((row: DurableSubscription) => row.subId),
+            );
+
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: `fs:${anchorUid}:list`,
+        });
+        const read = await readSubIds();
+        expect(read).toContain(plain.subId);
+        expect(read).not.toContain(schedule.subId);
+        expect(calls).toEqual([]);
+
+        page.mockClear();
+        await server.services.events.settleRevokedGrant({
+            holderUserId: userId,
+            appUid: APP_UID,
+            permission: EVENTS_BACKGROUND_PERMISSION,
+        });
+        expect(await readSubIds()).toContain(schedule.subId);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'suspended', reason: 'permission_revoked' },
+            },
+        ]);
+    });
+
+    it('tells the holder once per kind when an uninstall settles several pages', async () => {
+        recording();
+        const plains = [await plainRow(), await plainRow(), await plainRow()];
+        const schedules = [await scheduleRow(), await scheduleRow()];
+        const store = durable();
+        const listPage = store.listActiveForHolderPage.bind(store);
+        vi.spyOn(store, 'listActiveForHolderPage').mockImplementation(
+            (holder, app, afterId, _batchSize, opts) =>
+                listPage(holder, app, afterId, 1, opts),
+        );
+        const notify = vi.spyOn(server.services.notification, 'notify');
+
+        await expect(
+            server.services.events.settleRevokedGrant({
+                holderUserId: userId,
+                appUid: APP_UID,
+                permission: null,
+            }),
+        ).resolves.toBe(5);
+        const ended = notify.mock.calls
+            .filter(([, , meta]) => meta?.type === 'app.events.ended')
+            .map(([, body]) => body as { count: number; subjects: string[] });
+        expect(ended).toHaveLength(2);
+        expect(ended.map((body) => body.count).sort()).toEqual([2, 3]);
+        expect(ended.flatMap((body) => body.subjects).sort()).toEqual(
+            [...plains, ...schedules].map((row) => row.subject).sort(),
+        );
+    });
+
+    it('ends schedules app_deleted when their app is deleted', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await server.services.events.settleDeletedApp(APP_UID);
+        expect(calls).toEqual([
+            {
+                subIds: [schedule.subId],
+                change: { type: 'ended', endReason: 'app_deleted' },
+            },
+        ]);
+    });
+
+    it('ends schedules whose rows expire', async () => {
+        const calls = recording();
+        const schedule = await scheduleRow({ expiresAt: 1 });
+        await plainRow({ expiresAt: 1 });
+
+        await server.services.events.sweepExpired();
+        const handed = calls.flatMap((c) => c.subIds);
+        expect(handed).toEqual([schedule.subId]);
+        expect(calls[0].change).toEqual({ type: 'ended', endReason: 'ended' });
+    });
+
+    it('lets every path run without services.schedules', async () => {
+        setSchedules(undefined);
+        const schedule = await scheduleRow();
+        await plainRow();
+
+        await expect(
+            server.services.events.suspendSubscriptions(
+                [schedule],
+                'no_credit',
+            ),
+        ).resolves.toBe(1);
+        await expect(
+            server.services.events.resumeSubscriptions([
+                (await durable().getBySubId(schedule.subId))!,
+            ]),
+        ).resolves.toBe(1);
+        await server.services.events.settleDeletedApp(APP_UID);
+        const left = await server.clients.db.read(
+            'SELECT `token` FROM `event_subscriptions`',
+            [],
+        );
+        expect(
+            left.filter((r) => isScheduleRow(r as { token: string })),
+        ).toEqual([]);
+    });
+});

src/backend/services/events/workerSeam.ts

@@ -17,8 +17,14 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import type { EventsWorkerInvokerClient } from '../../clients/events/EventsWorkerInvokerClient.js';
-import type { DeliverableEvent } from './registry.js';
+import type {
+    EventsWorkerInvokerClient,
+    WorkerInvokeOptions,
+} from '../../clients/events/EventsWorkerInvokerClient.js';
+import type { Actor } from '../../core/actor.js';
+import type { DeliverableEvent, ScheduleEvent } from './registry.js';
+
+export type { ScheduleEvent } from './registry.js';
 
 /**
  * Where a delivery leaves the event system for the app's own code.
@@ -36,11 +42,13 @@ export interface WorkerInvocation {
     /** The app whose handler runs, or null for an account-owned row. */
     appUid: string | null;
     handlerName: string | null;
-    event: DeliverableEvent;
+    event: DeliverableEvent | ScheduleEvent;
     /** The subscription's stored context, delivered to the handler as `ctx`. */
     context: string | null;
     /** Handler runs behind `event`; the token this run is handed is one deeper. */
     handlerDepth: number;
+    /** The holder acting as the app, when the caller resolved it already. */
+    actor?: Actor;
 }
 
 /**
@@ -50,7 +58,11 @@ export interface WorkerInvocation {
  * toward the run that suspends a subscription.
  */
 export type WorkerInvocationOutcome =
-    'settled' | 'terminal' | 'retriable' | 'unavailable' | 'deferred';
+    | 'settled'
+    | 'terminal'
+    | 'retriable'
+    | 'unavailable'
+    | 'deferred';
 
 export interface WorkerInvocationResult {
     outcome: WorkerInvocationOutcome;
@@ -64,6 +76,12 @@ export interface WorkerInvokerSeam {
     available?(): boolean;
 }
 
+/**
+ * Why no token could be minted: nobody to run as, no consent to, or the mint
+ * itself failed.
+ */
+export type MintRefusal = 'no-identity' | 'no-consent' | 'mint-failed';
+
 /**
  * Mints the subscriber's app-under-user token for one invocation. The
  * `events:background` consent is what authorizes it, not a narrower
@@ -71,7 +89,14 @@ export interface WorkerInvokerSeam {
  */
 export type SubscriberTokenMinter = (
     invocation: WorkerInvocation,
-) => Promise<string | null>;
+) => Promise<{ token: string } | { refused: MintRefusal }>;
+
+/** Where the handler set is read from when its cache misses. */
+export interface AddressOptions {
+    missRead?: 'primary' | 'replica';
+    /** A handler the set must have; a set without it is re-read on the primary. */
+    handlerName?: string;
+}
 
 /**
  * The script an app's handlers deploy as and its invocation key; null when the
@@ -79,8 +104,17 @@ export type SubscriberTokenMinter = (
  */
 export type EventsWorkerAddresser = (
     appUid: string,
+    options?: AddressOptions,
 ) => Promise<{ script: string; key: string } | null>;
 
+/** What one run came to, including why nothing was invoked. */
+export type WorkerRunResult =
+    | WorkerInvocationResult
+    | { outcome: MintRefusal | 'deploy-busy' };
+
+/** How {@link EventsWorkerInvoker.run} reaches the worker. */
+export type WorkerRunOptions = WorkerInvokeOptions & AddressOptions;
+
 /** How often the same script's retriable failures are logged. */
 const WARN_INTERVAL_MS = 60_000;
 /** Distinct scripts one process tracks a last-warned time for. */
@@ -122,28 +156,57 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
     async invoke(
         invocation: WorkerInvocation,
     ): Promise<WorkerInvocationResult> {
+        const result = await this.run(invocation);
+        switch (result.outcome) {
+            // Nothing was attempted.
+            case 'no-identity':
+            case 'no-consent':
+            case 'mint-failed':
+                return { outcome: 'deferred' };
+            case 'deploy-busy':
+                return { outcome: 'unavailable' };
+            default:
+                return result;
+        }
+    }
+
+    /** {@link invoke}, saying why nothing was invoked. */
+    async run(
+        invocation: WorkerInvocation,
+        options: WorkerRunOptions = {},
+    ): Promise<WorkerRunResult> {
         const { appUid, handlerName } = invocation;
         if (!appUid || !handlerName) return { outcome: 'deferred' };
         if (!this.#available()) return { outcome: 'unavailable' };
 
-        const token = await this.#mintToken(invocation);
-        if (token === null) return { outcome: 'deferred' };
+        const minted = await this.#mintToken(invocation);
+        if ('refused' in minted) return { outcome: minted.refused };
 
         // Nowhere to send it: the app has no handlers or no owner to run them
         // as. Retrying cannot fix that, so the consecutive-failure rule is what
         // stops it.
-        const address = await this.#address(appUid);
+        const address = await this.#address(appUid, {
+            missRead: options.missRead,
+            handlerName,
+        });
         if (address === null) return { outcome: 'retriable' };
 
-        const result = await this.#client.invoke({
-            script: address.script,
-            key: address.key,
-            appUid,
-            handler: handlerName,
-            token,
-            event: invocation.event,
-            ctx: parseContext(invocation.context),
-        });
+        const result = await this.#client.invoke(
+            {
+                script: address.script,
+                key: address.key,
+                appUid,
+                handler: handlerName,
+                token: minted.token,
+                event: invocation.event,
+                ctx: parseContext(invocation.context),
+            },
+            {
+                lane: options.lane,
+                stampEvent: options.stampEvent,
+            },
+        );
+        if (result.deployBusy) return { outcome: 'deploy-busy' };
         if (
             (result.outcome === 'retriable' ||
                 result.outcome === 'unavailable') &&

src/backend/services/index.ts

@@ -36,6 +36,7 @@ import { FSService } from './fs/FSService';
 import { KvShareService } from './kvShare/KvShareService';
 import { ServerHealthService } from './health/ServerHealthService';
 import { PuterHomepageService } from './homepage/PuterHomepageService';
+import { JobScheduler } from './jobs/JobScheduler';
 import { LocalWorkerService } from './localworker/LocalWorkerService';
 import { MeteringService } from './metering/MeteringService';
 import { NotificationService } from './notification/NotificationService';
@@ -60,6 +61,7 @@ import { UserAccountService } from './user/UserAccountService';
  */
 declare module './types' {
     interface IPuterServiceInstances {
+        jobs: JobScheduler;
         metering: MeteringService;
         appOriginBlocklist: AppOriginBlocklistService;
         permission: PermissionService;
@@ -103,6 +105,9 @@ declare module './types' {
 // NotificationService depends on notification store (for DB) + event client (for socket push).
 // BroadcastService is independent — only needs the event client.
 export const puterServices = {
+    // First, so every later service can register jobs with it. Reaches
+    // `broadcast` at call time only.
+    jobs: JobScheduler,
     metering: MeteringService,
     // Declared before `auth` so AuthService sees it as a prior peer — it
     // queries the blocklist on app-token acquisition and per-request app

src/backend/services/jobs/JobScheduler.test.ts

@@ -0,0 +1,781 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import MockRedis from 'ioredis-mock';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobLeaseLostError,
+    JobScheduler,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const HOUR = 60 * 60_000;
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+describe('JobScheduler', () => {
+    let server: PuterServer;
+    const schedulers: JobScheduler[] = [];
+
+    beforeAll(async () => {
+        server = await setupTestServer();
+    });
+
+    afterAll(async () => {
+        await server?.shutdown();
+    });
+
+    afterEach(() => {
+        vi.restoreAllMocks();
+        vi.useRealTimers();
+        for (const scheduler of schedulers.splice(0)) {
+            scheduler.onServerPrepareShutdown();
+        }
+    });
+
+    const db = () => server.clients.db;
+
+    /**
+     * A node of its own: its own config and region, on the test server's Redis
+     * unless given one.
+     */
+    const node = ({
+        namespace,
+        region = 'r1',
+        redis,
+        domain = 'puter.test',
+        enabled,
+    }: {
+        namespace?: string;
+        region?: string;
+        redis?: InstanceType<typeof MockRedis>;
+        domain?: string;
+        enabled?: boolean;
+    } = {}) => {
+        const config = {
+            port: 0,
+            extensions: [],
+            domain,
+            serverId: `node-${schedulers.length}`,
+            jobs: { namespace, enabled },
+        } as IConfig;
+        const clients = (
+            redis ? { ...server.clients, redis } : server.clients
+        ) as SchedulerArgs[1];
+        const services = {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3];
+        const scheduler = new JobScheduler(
+            config,
+            clients,
+            server.stores as SchedulerArgs[2],
+            services,
+        );
+        schedulers.push(scheduler);
+        return scheduler;
+    };
+
+    const freshNamespace = () => `ns-${randomUUID()}`;
+
+    /**
+     * A job that runs `batches` batches `batchMs` apart, checking its signal
+     * before each.
+     */
+    const batchedJob = (batches: number, batchMs: number) => {
+        const seen = { done: 0, signal: null as AbortSignal | null };
+        const run = async ({ signal }: JobRunContext) => {
+            seen.signal = signal;
+            for (let i = 0; i < batches; i++) {
+                if (signal.aborted) return;
+                await sleep(batchMs);
+                seen.done += 1;
+            }
+        };
+        return { seen, run };
+    };
+
+    const keysOf = (namespace: string, name: string) => ({
+        lock: `jobs:{${namespace}:${name}}:lock`,
+        gate: `jobs:{${namespace}:${name}}:next`,
+        run: `jobs:{${namespace}:${name}}:run`,
+    });
+
+    // -- Registration -------------------------------------------------
+
+    it('refuses a bad or duplicate registration', () => {
+        const scheduler = node({ namespace: freshNamespace() });
+        const run = async () => {};
+        expect(() =>
+            scheduler.register('bad name', {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job', { every: 0, scope: 'fleet', run }),
+        ).toThrow(/every/);
+        expect(() =>
+            scheduler.register('job', {
+                every: 1000,
+                scope: 'fleet',
+                leaseMs: 1.5,
+                run,
+            }),
+        ).toThrow(/leaseMs/);
+        // `:` would let two (namespace, name) pairs share Redis keys.
+        expect(() =>
+            scheduler.register('b:c', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('x'.repeat(129), {
+                every: 1000,
+                scope: 'fleet',
+                run,
+            }),
+        ).toThrow(/invalid job name/);
+        expect(() =>
+            scheduler.register('job.region', {
+                every: 1000,
+                scope: 'region',
+                leaseMs: 2000,
+                run,
+            }),
+        ).toThrow(/exceeds every/);
+        scheduler.register('job', { every: 1000, scope: 'fleet', run });
+        expect(() =>
+            scheduler.register('job', { every: 1000, scope: 'fleet', run }),
+        ).toThrow(/already registered/);
+    });
+
+    it('falls back to the default namespace, once and loudly, when the configured one cannot be held', () => {
+        const run = async () => {};
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        for (const namespace of ['a:b', 'x'.repeat(129), 'spaced name']) {
+            const scheduler = node({ namespace });
+            scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+            expect(scheduler.namespace).toBe('default');
+        }
+        logged.mockClear();
+
+        // A self-host whose domain carries a port still boots.
+        const scheduler = node({ domain: 'localhost:4100' });
+        scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
+        scheduler.register('job.b', { every: 1000, scope: 'fleet', run });
+        expect(scheduler.namespace).toBe('default');
+        expect(logged).toHaveBeenCalledTimes(1);
+        expect(String(logged.mock.calls[0][0])).toMatch(
+            /localhost:4100.*jobs\.namespace/,
+        );
+    });
+
+    it('does nothing at all while no job is registered', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        // Timers the test runtime itself already holds.
+        const baseline = vi.getTimerCount();
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        const scheduler = node({ namespace: freshNamespace() });
+
+        scheduler.onServerStart();
+        expect(vi.getTimerCount()).toBe(baseline);
+        await scheduler.poll();
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+
+        // A job registered after start arms the poll; shutdown disarms it.
+        scheduler.register('late', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {},
+        });
+        expect(vi.getTimerCount()).toBeGreaterThan(baseline);
+        scheduler.onServerPrepareShutdown();
+        expect(vi.getTimerCount()).toBe(baseline);
+    });
+
+    it('claims nothing once draining, and leaves a running job its lease', async () => {
+        const scheduler = node({ namespace: `drain${Date.now()}` });
+        let release!: () => void;
+        let signal!: AbortSignal;
+        scheduler.register('long', {
+            every: HOUR,
+            scope: 'region',
+            run: (ctx) => {
+                signal = ctx.signal;
+                return new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+            },
+        });
+        const first = scheduler.poll();
+        await vi.waitFor(() => expect(release).toBeDefined());
+        const run = vi.fn(async () => {});
+        scheduler.register('other', { every: HOUR, scope: 'region', run });
+
+        scheduler.onServerDrainStart();
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(signal.aborted).toBe(false);
+
+        release();
+        await first;
+    });
+
+    it('arms no poll on a node with jobs turned off, though poll() still runs them', async () => {
+        vi.useFakeTimers({
+            toFake: [
+                'setInterval',
+                'setTimeout',
+                'clearInterval',
+                'clearTimeout',
+            ],
+        });
+        const baseline = vi.getTimerCount();
+        const scheduler = node({ namespace: freshNamespace(), enabled: false });
+        const warned = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let runs = 0;
+        scheduler.onServerStart();
+        scheduler.register('job.off', {
+            every: HOUR,
+            scope: 'fleet',
+            run: async () => {
+                runs += 1;
+            },
+        });
+        scheduler.register('job.off.too', {
+            every: HOUR,
+            scope: 'region',
+            run: async () => undefined,
+        });
+        expect(vi.getTimerCount()).toBe(baseline);
+        // Said once per node, not per job.
+        expect(
+            warned.mock.calls.filter(([line]) =>
+                String(line).includes('jobs.enabled'),
+            ),
+        ).toHaveLength(1);
+        vi.useRealTimers();
+
+        await scheduler.poll();
+        expect(runs).toBe(1);
+    });
+
+    describe('on a controlled clock', () => {
+        const leaseMs = 300;
+        /**
+         * Pending timeouts armed from JobScheduler.ts. The shared test server's
+         * own loops keep running on real intervals and may arm fake timers
+         * mid-test, so the global timer count says nothing about the
+         * scheduler.
+         */
+        let armed: Set<unknown>;
+        const schedulerTimers = () => armed.size;
+
+        beforeEach(() => {
+            vi.useFakeTimers({
+                toFake: [
+                    'setTimeout',
+                    'clearTimeout',
+                    'setInterval',
+                    'clearInterval',
+                    'performance',
+                ],
+            });
+            armed = new Set();
+            const fakeSetTimeout = globalThis.setTimeout;
+            const fakeClearTimeout = globalThis.clearTimeout;
+            const fromScheduler = () => {
+                const limit = Error.stackTraceLimit;
+                Error.stackTraceLimit = 30;
+                const stack = new Error().stack ?? '';
+                Error.stackTraceLimit = limit;
+                return /services[\\/]jobs[\\/]JobScheduler\.ts/.test(stack);
+            };
+            vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+                fn: (...args: unknown[]) => void,
+                ms?: number,
+                ...rest: unknown[]
+            ) => {
+                if (!fromScheduler()) return fakeSetTimeout(fn, ms, ...rest);
+                const handle = fakeSetTimeout(
+                    (...args: unknown[]) => {
+                        armed.delete(handle);
+                        fn(...args);
+                    },
+                    ms,
+                    ...rest,
+                );
+                armed.add(handle);
+                return handle;
+            }) as never);
+            vi.spyOn(globalThis, 'clearTimeout').mockImplementation(((
+                handle: Parameters<typeof clearTimeout>[0],
+            ) => {
+                armed.delete(handle);
+                fakeClearTimeout(handle);
+            }) as never);
+        });
+
+        /** Batches of `batchMs`, each started only while the signal is live. */
+        const timedJob = (batches: number, batchMs: number) => {
+            const seen = {
+                starts: [] as number[],
+                abortedAt: null as number | null,
+                signal: null as AbortSignal | null,
+                saveError: null as unknown,
+            };
+            const run = async ({ signal, saveState }: JobRunContext) => {
+                seen.signal = signal;
+                signal.addEventListener('abort', () => {
+                    seen.abortedAt = performance.now();
+                });
+                for (let i = 0; i < batches; i++) {
+                    if (signal.aborted) break;
+                    seen.starts.push(performance.now());
+                    await sleep(batchMs);
+                }
+                try {
+                    await saveState({ late: true });
+                } catch (err) {
+                    seen.saveError = err;
+                }
+            };
+            return { seen, run };
+        };
+
+        /** Let the run play out, then check nothing it armed is left behind. */
+        const settle = async (running: Promise<void>) => {
+            await vi.advanceTimersByTimeAsync(2_000);
+            await running;
+            expect(schedulerTimers()).toBe(0);
+        };
+
+        it('clears a run’s timers at shutdown even while the run hangs', async () => {
+            const scheduler = node({ namespace: freshNamespace() });
+            let release!: () => void;
+            scheduler.register('job.hang', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: () =>
+                    new Promise<void>((resolve) => {
+                        release = resolve;
+                    }),
+            });
+            const running = scheduler.poll();
+            await vi.advanceTimersByTimeAsync(10);
+            expect(schedulerTimers()).toBeGreaterThan(0);
+
+            scheduler.onServerPrepareShutdown();
+            expect(schedulerTimers()).toBe(0);
+            release();
+            await settle(running);
+        });
+
+        it('never starts a region run once taking the lock used up half the lease', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17104 });
+            const scheduler = node({ namespace, redis });
+            const run = vi.fn(async () => {});
+            scheduler.register('job.slowLock', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run,
+            });
+            const set = redis.set.bind(redis) as (
+                ...args: unknown[]
+            ) => Promise<unknown>;
+            const slowLock = vi.spyOn(redis, 'set').mockImplementation((async (
+                ...args: unknown[]
+            ) => {
+                const result = await set(...args);
+                if (String(args[0]).endsWith(':lock')) {
+                    await sleep(leaseMs / 2 + 10);
+                }
+                return result;
+            }) as never);
+            await settle(scheduler.poll());
+
+            expect(run).not.toHaveBeenCalled();
+            // The gate went back, so the next poll runs it.
+            expect(
+                await redis.exists(`jobs:{${namespace}:job.slowLock}:next`),
+            ).toBe(0);
+            slowLock.mockRestore();
+            await settle(scheduler.poll());
+            expect(run).toHaveBeenCalledTimes(1);
+        });
+
+        it('aborts a region run before its next batch once the lock lapses, and leaves it due at once', async () => {
+            const namespace = freshNamespace();
+            const redis = new MockRedis({ port: 17106 });
+            const scheduler = node({ namespace, redis });
+            const job = timedJob(10, 70);
+            scheduler.register('job.lapse', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs,
+                run: job.run,
+            });
+            const lockedAt = performance.now();
+            await settle(scheduler.poll());
+
+            expect(job.seen.abortedAt! - lockedAt).toBe(leaseMs);
+            expect((job.seen.signal?.reason as Error).message).toMatch(
+                /lapsed/,
+            );
+            expect(job.seen.starts.length).toBeLessThan(10);
+            expect(job.seen.starts.every((t) => t < job.seen.abortedAt!)).toBe(
+                true,
+            );
+            // The gate is still this run's, so its last save went through;
+            // interrupted, the gate goes back and the job is due at once.
+            expect(job.seen.saveError).toBeNull();
+            const keys = {
+                gate: `jobs:{${namespace}:job.lapse}:next`,
+                run: `jobs:{${namespace}:job.lapse}:run`,
+            };
+            expect(await redis.exists(keys.gate)).toBe(0);
+            expect(await redis.hget(keys.run, 'state')).toBe('{"late":true}');
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        });
+    });
+
+    it('refuses state over 64 KB', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17107 });
+        const scheduler = node({ namespace, redis });
+        let refused: unknown = null;
+        scheduler.register('job.bigState', {
+            every: HOUR,
+            scope: 'region',
+            run: async ({ saveState }) => {
+                try {
+                    await saveState({ blob: 'x'.repeat(70_000) });
+                } catch (err) {
+                    refused = err;
+                }
+            },
+        });
+        await scheduler.poll();
+        expect((refused as Error).message).toMatch(/over 65536 bytes/);
+        expect(
+            await redis.hget(keysOf(namespace, 'job.bigState').run, 'state'),
+        ).toBeNull();
+    });
+
+    it('releases an interrupted run on shutdown without counting a failure', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17109 });
+        const scheduler = node({ namespace, redis });
+        const job = batchedJob(50, 50);
+        scheduler.register('job.shutdown', {
+            every: HOUR,
+            scope: 'region',
+            run: job.run,
+        });
+        const running = scheduler.poll();
+        await vi.waitFor(() => expect(job.seen.done).toBeGreaterThan(0));
+        scheduler.onServerPrepareShutdown();
+        await running;
+
+        expect(job.seen.done).toBeLessThan(50);
+        expect((job.seen.signal?.reason as Error).message).toBe(
+            'server shutting down',
+        );
+        const keys = keysOf(namespace, 'job.shutdown');
+        // Due again at once, with no failure counted.
+        expect(await redis.exists(keys.gate)).toBe(0);
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+
+    it('keeps namespaces apart: the same job name runs once in each', async () => {
+        const redis = new MockRedis({ port: 17110 });
+        const [nsA, nsB] = [freshNamespace(), freshNamespace()];
+        const nodes = [
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsA, redis }),
+            node({ namespace: nsB, redis }),
+            node({ namespace: nsB, redis }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.shared', {
+                every: HOUR,
+                scope: 'region',
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+    });
+
+    it('falls back to the domain as its namespace', () => {
+        expect(node({ domain: 'example.test' }).namespace).toBe('example.test');
+        expect(
+            node({ namespace: 'explicit', domain: 'example.test' }).namespace,
+        ).toBe('explicit');
+    });
+
+    it('boots a server whose domain carries a port, registering its jobs under the default namespace', async () => {
+        const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const portDomain = await setupTestServer({
+            domain: 'localhost:4101',
+        } as IConfig);
+        try {
+            expect(portDomain.services.jobs.namespace).toBe('default');
+            const namespaceErrors = logged.mock.calls.filter(([message]) =>
+                String(message).startsWith('[jobs] namespace'),
+            );
+            expect(namespaceErrors).toHaveLength(1);
+        } finally {
+            await portDomain.shutdown();
+        }
+    });
+
+    // -- Region scope -------------------------------------------------
+
+    it('runs a region job once per region', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17101 });
+        const redisB = new MockRedis({ port: 17102 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const runs: string[] = [];
+        nodes.forEach((scheduler, i) =>
+            scheduler.register('job.region', {
+                every: HOUR,
+                scope: 'region',
+                leaseMs: 5_000,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    await sleep(20);
+                },
+            }),
+        );
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        expect(
+            await redisA.pttl(`jobs:{${namespace}:job.region}:next`),
+        ).toBeGreaterThan(HOUR - 60_000);
+        // The region lock was released after the run.
+        expect(await redisA.exists(`jobs:{${namespace}:job.region}:lock`)).toBe(
+            0,
+        );
+    });
+
+    it('backs off a failing region job and keeps its state in Redis', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17103 });
+        const scheduler = node({ namespace, redis });
+        const seen: unknown[] = [];
+        let fail = true;
+        scheduler.register('job.regionFail', {
+            every: HOUR,
+            scope: 'region',
+            leaseMs: 5_000,
+            run: async ({ state, saveState }) => {
+                seen.push(state);
+                await saveState({ page: seen.length });
+                if (fail) throw new Error('nope');
+            },
+        });
+        const gate = `jobs:{${namespace}:job.regionFail}:next`;
+
+        await scheduler.poll();
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBe('1');
+        expect(await redis.pttl(gate)).toBeLessThanOrEqual(2_000);
+
+        fail = false;
+        await redis.del(gate);
+        await scheduler.poll();
+        expect(seen).toEqual([null, { page: 1 }]);
+        expect(
+            await redis.hget(
+                `jobs:{${namespace}:job.regionFail}:run`,
+                'failures',
+            ),
+        ).toBeNull();
+        expect(await redis.pttl(gate)).toBeGreaterThan(HOUR - 60_000);
+    });
+
+    it('skips region jobs quietly while Redis is unreachable', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17108 });
+        const scheduler = node({ namespace, redis });
+        const run = vi.fn(async () => {});
+        scheduler.register('job.noRedis', {
+            every: HOUR,
+            scope: 'region',
+            run,
+        });
+        vi.spyOn(redis, 'exists').mockRejectedValue(
+            new Error('connection refused'),
+        );
+        const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+        await scheduler.poll();
+        expect(run).not.toHaveBeenCalled();
+        expect(errors).not.toHaveBeenCalled();
+    });
+
+    it('fences a region run that outlived its gate off its successor’s keys', async () => {
+        const namespace = freshNamespace();
+        const redis = new MockRedis({ port: 17105 });
+        const [slow, next] = [
+            node({ namespace, redis }),
+            node({ namespace, redis }),
+        ];
+        const keys = {
+            gate: `jobs:{${namespace}:job.fenced}:next`,
+            run: `jobs:{${namespace}:job.fenced}:run`,
+        };
+        let release!: () => void;
+        let slowStarted = false;
+        let slowSaveError: unknown = null;
+        // Ignores its signal and outlives both its 200 ms lock and 400 ms gate.
+        slow.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                slowStarted = true;
+                await new Promise<void>((resolve) => {
+                    release = resolve;
+                });
+                try {
+                    await saveState({ by: 'slow' });
+                } catch (err) {
+                    slowSaveError = err;
+                }
+                throw new Error('slow run failed');
+            },
+        });
+        next.register('job.fenced', {
+            every: 400,
+            scope: 'region',
+            leaseMs: 200,
+            run: async ({ saveState }) => {
+                await saveState({ by: 'next' });
+            },
+        });
+
+        const slowRun = slow.poll();
+        await vi.waitFor(() => expect(slowStarted).toBe(true));
+        await sleep(450);
+        await next.poll();
+        const successorGate = await redis.get(keys.gate);
+        expect(successorGate).not.toBeNull();
+
+        release();
+        await slowRun;
+        expect(slowSaveError).toBeInstanceOf(JobLeaseLostError);
+        // Neither the slow run's state, its failure count, nor its cleanup
+        // touched what the successor holds.
+        expect(await redis.get(keys.gate)).toBe(successorGate);
+        expect(await redis.pttl(keys.gate)).toBeGreaterThan(0);
+        expect(await redis.hget(keys.run, 'state')).toBe('{"by":"next"}');
+        expect(await redis.hget(keys.run, 'failures')).toBeNull();
+    });
+    it('runs a fleet job once per region, its lease capped at its cadence and nearPrimary ignored', async () => {
+        const namespace = freshNamespace();
+        const redisA = new MockRedis({ port: 17111 });
+        const redisB = new MockRedis({ port: 17112 });
+        const nodes = [
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'a', redis: redisA }),
+            node({ namespace, region: 'b', redis: redisB }),
+            node({ namespace, region: 'b', redis: redisB }),
+        ];
+        const keys = keysOf(namespace, 'job.fleet');
+        const runs: string[] = [];
+        const lockTtls: number[] = [];
+        const reads = vi.spyOn(db(), 'read');
+        const preads = vi.spyOn(db(), 'pread');
+        const writes = vi.spyOn(db(), 'write');
+        nodes.forEach((scheduler, i) => {
+            const redis = i < 2 ? redisA : redisB;
+            scheduler.register('job.fleet', {
+                every: 2_000,
+                scope: 'fleet',
+                nearPrimary: true,
+                // Longer than `every`: refused for a region job, capped here.
+                leaseMs: HOUR,
+                run: async () => {
+                    runs.push(i < 2 ? 'a' : 'b');
+                    lockTtls.push(await redis.pttl(keys.lock));
+                    await sleep(20);
+                },
+            });
+        });
+
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+        expect(runs.sort()).toEqual(['a', 'b']);
+        for (const ttl of lockTtls) {
+            expect(ttl).toBeGreaterThan(0);
+            expect(ttl).toBeLessThanOrEqual(2_000);
+        }
+        expect(await redisA.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(await redisB.pttl(keys.gate)).toBeGreaterThan(1_000);
+        expect(reads).not.toHaveBeenCalled();
+        expect(preads).not.toHaveBeenCalled();
+        expect(writes).not.toHaveBeenCalled();
+    });
+});

src/backend/services/jobs/JobScheduler.ts

@@ -0,0 +1,528 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { withRedisLock } from '../../util/redisLock.js';
+import { resolveJobsNamespace } from '../../util/jobsNamespace.js';
+import { deliveryBackoffMs } from '../../util/retryBackoff.js';
+import { PuterService } from '../types';
+import {
+    JobLeaseLostError,
+    asError,
+    lostLease,
+    parseState,
+    runOutcome,
+    serializeState,
+    type RegisteredJob,
+    type RunLease,
+} from './jobRun.js';
+
+export { JobLeaseLostError } from './jobRun.js';
+
+/** How often each node looks for due jobs. */
+const JOB_POLL_MS = 30_000;
+/** Each poll waits up to this long first, so nodes don't all claim at once. */
+const JOB_POLL_JITTER_MS = 5_000;
+const JOB_DEFAULT_LEASE_MS = 15 * 60_000;
+/** Consecutive failures that raise `scheduled_job_failing`. */
+const JOB_FAILING_ALARM_AT = 3;
+const JOB_SHUTDOWN_WAIT_MS = 10_000;
+/**
+ * Job names: ASCII, at most 128, and without `:`, so a (namespace, name) pair
+ * maps to exactly one set of Redis keys (namespaces follow the same rule).
+ */
+const JOB_ID_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+// Region writes apply only while the gate still holds this run's token.
+const REGION_FENCED_WRITE = `
+if redis.call('get', KEYS[1]) ~= ARGV[1] then return -1 end
+local op = ARGV[2]
+if op == 'hset' then return redis.call('hset', KEYS[2], ARGV[3], ARGV[4]) end
+if op == 'hdel' then return redis.call('hdel', KEYS[2], ARGV[3]) end
+if op == 'hincrby' then return redis.call('hincrby', KEYS[2], ARGV[3], 1) end
+if op == 'pexpire' then return redis.call('pexpire', KEYS[1], ARGV[3]) end
+if op == 'del' then return redis.call('del', KEYS[1]) end
+return -1
+`;
+
+/**
+ * `region` runs once per cadence per region. `fleet` means once across every
+ * region where `services.jobs` supports it; this default runs it per region.
+ */
+export type JobScope = 'fleet' | 'region';
+
+export interface JobRunContext {
+    /** Aborts once the run may have lost the job. Check it before each batch. */
+    signal: AbortSignal;
+    /** What an earlier run saved, or null. */
+    state: unknown;
+    /**
+     * Persist JSON-serializable `state` for later runs. Throws once the run
+     * lost the job.
+     */
+    saveState: (state: unknown) => Promise<void>;
+}
+
+export interface JobDefinition {
+    /** Cadence in ms. */
+    every: number;
+    scope: JobScope;
+    /**
+     * Prefer the region nearest the primary database, for a `fleet` job. The
+     * default scheduler ignores it.
+     */
+    nearPrimary?: boolean;
+    /**
+     * How long a run holds the job, in total on the region path (at most
+     * `every`). Default 15 min, capped at `every` for `region`; a `fleet` job
+     * run per region is capped at `every` too.
+     */
+    leaseMs?: number;
+    run: (ctx: JobRunContext) => Promise<void>;
+}
+
+/** The `afterId` cursor a keyset-walking job saved in its state, or 0. */
+export const savedAfterId = (state: unknown): number => {
+    const afterId = (state as { afterId?: unknown } | null)?.afterId;
+    return typeof afterId === 'number' &&
+        Number.isSafeInteger(afterId) &&
+        afterId > 0
+        ? afterId
+        : 0;
+};
+
+interface RegionKeys {
+    lock: string;
+    gate: string;
+    run: string;
+}
+
+/**
+ * Runs our own recurring backend work once per cadence per region, under a
+ * Redis lock. `fleet` jobs take the same path here; an extension registered as
+ * `services.jobs` can run them fleet-wide by overriding `pollFleet`. Nothing
+ * polls until a job is registered.
+ */
+export class JobScheduler extends PuterService {
+    #jobs = new Map<string, RegisteredJob>();
+    #running = new Map<string, Promise<void>>();
+    #leases = new Set<RunLease>();
+    #started = false;
+    #stopping = false;
+    /** Polling stopped; runs already going keep their leases. */
+    #draining = false;
+    #timer: ReturnType<typeof setInterval> | null = null;
+    #pollDelays = new Set<ReturnType<typeof setTimeout>>();
+    #warnedOff = false;
+    #namespace: string | null = null;
+
+    /** Keys this deployment's jobs, so deployments sharing Redis stay apart. */
+    get namespace(): string {
+        return (this.#namespace ??= resolveJobsNamespace(this.config));
+    }
+
+    register(name: string, definition: JobDefinition): void {
+        if (!JOB_ID_PATTERN.test(name)) {
+            throw new Error(`invalid job name: ${name}`);
+        }
+        if (this.#jobs.has(name)) {
+            throw new Error(`job already registered: ${name}`);
+        }
+        const { every, scope } = definition;
+        if (!Number.isSafeInteger(every) || every <= 0) {
+            throw new Error(`job ${name}: every must be a positive integer`);
+        }
+        if (scope !== 'fleet' && scope !== 'region') {
+            throw new Error(`job ${name}: unknown scope`);
+        }
+        const leaseMs =
+            definition.leaseMs ??
+            (scope === 'region'
+                ? Math.min(JOB_DEFAULT_LEASE_MS, every)
+                : JOB_DEFAULT_LEASE_MS);
+        if (!Number.isSafeInteger(leaseMs) || leaseMs <= 0) {
+            throw new Error(`job ${name}: leaseMs must be a positive integer`);
+        }
+        // The gate that fences a region run lasts `every`.
+        if (scope === 'region' && leaseMs > every) {
+            throw new Error(
+                `job ${name}: a region job's leaseMs exceeds every`,
+            );
+        }
+        this.#jobs.set(name, {
+            every,
+            scope,
+            nearPrimary: definition.nearPrimary ?? false,
+            leaseMs,
+            run: definition.run,
+        });
+        if (this.#started) this.#arm();
+    }
+
+    override onServerStart(): void {
+        this.#started = true;
+        if (this.#jobs.size > 0) this.#arm();
+    }
+
+    /** Claim nothing new while draining; running jobs carry on until prepare. */
+    override onServerDrainStart(): void {
+        this.#stopPolling();
+    }
+
+    override onServerPrepareShutdown(): void {
+        this.#stopping = true;
+        this.#stopPolling();
+        for (const lease of [...this.#leases]) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+            lease.close();
+        }
+    }
+
+    #stopPolling(): void {
+        this.#draining = true;
+        if (this.#timer) clearInterval(this.#timer);
+        this.#timer = null;
+        for (const delay of this.#pollDelays) clearTimeout(delay);
+        this.#pollDelays.clear();
+    }
+
+    /**
+     * Give aborted runs a moment to record themselves before the database
+     * closes.
+     */
+    override async onServerShutdown(): Promise<void> {
+        if (this.#running.size === 0) return;
+        let wait: ReturnType<typeof setTimeout> | undefined;
+        await Promise.race([
+            Promise.allSettled(this.#running.values()),
+            new Promise((resolve) => {
+                wait = setTimeout(resolve, JOB_SHUTDOWN_WAIT_MS);
+            }),
+        ]);
+        clearTimeout(wait);
+    }
+
+    /**
+     * Start every registered job that is due and not already running here.
+     * Resolves once the runs it started have finished.
+     */
+    async poll(): Promise<void> {
+        if (this.#stopping || this.#draining) return;
+        const fleet: string[] = [];
+        const region: Promise<void>[] = [];
+        for (const [name, job] of this.#jobs) {
+            if (this.#running.has(name)) continue;
+            if (job.scope === 'fleet') {
+                fleet.push(name);
+                continue;
+            }
+            region.push(this.track(name, () => this.#tryRegion(name, job)));
+        }
+        await Promise.all([this.pollFleet(fleet), ...region]);
+    }
+
+    #arm(): void {
+        if (this.#timer || this.#stopping || this.#draining) return;
+        if (this.config.jobs?.enabled === false) {
+            // Nothing alarms on a job that never runs, so say it here.
+            if (!this.#warnedOff) {
+                console.warn(
+                    '[jobs] polling is off on this node (jobs.enabled: false); other nodes must run the jobs',
+                );
+            }
+            this.#warnedOff = true;
+            return;
+        }
+        const tick = () => {
+            const delay = setTimeout(() => {
+                this.#pollDelays.delete(delay);
+                void this.#pollSafely();
+            }, Math.random() * JOB_POLL_JITTER_MS);
+            delay.unref?.();
+            this.#pollDelays.add(delay);
+        };
+        const timer = setInterval(tick, JOB_POLL_MS);
+        timer.unref?.();
+        this.#timer = timer;
+        tick();
+    }
+
+    async #pollSafely(): Promise<void> {
+        try {
+            await this.poll();
+        } catch (err) {
+            console.error('[jobs] poll failed', err);
+        }
+    }
+
+    // -- Region scope -------------------------------------------------
+
+    async #tryRegion(name: string, job: RegisteredJob): Promise<void> {
+        const redis = this.clients.redis;
+        const keys = regionKeys(this.namespace, name);
+        let gated: number;
+        try {
+            gated = await redis.exists(keys.gate);
+        } catch {
+            // Redis is unreachable; the lock below would be busy anyway.
+            return;
+        }
+        if (gated > 0) return;
+        // Before the lock, so acquiring it counts against `leaseMs`.
+        const startedAt = performance.now();
+        await withRedisLock(
+            redis,
+            keys.lock,
+            async () => {
+                const token = randomUUID();
+                const gate = await redis.set(
+                    keys.gate,
+                    token,
+                    'PX',
+                    job.every,
+                    'NX',
+                );
+                if (gate !== 'OK') return;
+                await this.#runRegion(name, job, keys, token, startedAt);
+            },
+            {
+                ttlMs: job.leaseMs,
+                onUnavailable: 'busy',
+                onBusy: (): void => undefined,
+            },
+        );
+    }
+
+    /**
+     * One run under the region lock. The lock doesn't renew, so the signal
+     * aborts once it lapses, and every write is fenced on the gate's token.
+     */
+    async #runRegion(
+        name: string,
+        job: RegisteredJob,
+        keys: RegionKeys,
+        token: string,
+        startedAt: number,
+    ): Promise<void> {
+        const lease = this.#regionLease(job.leaseMs, startedAt);
+        const { signal } = lease.controller;
+        const write = (op: string, ...args: (string | number)[]) =>
+            this.#regionWrite(keys, token, op, ...args);
+        let error: unknown = null;
+        try {
+            const saved = await this.clients.redis.hget(keys.run, 'state');
+            if (await lease.ready()) {
+                await job.run({
+                    signal,
+                    state: parseState(saved),
+                    saveState: async (state) => {
+                        const json = serializeState(state);
+                        if ((await write('hset', 'state', json)) < 0) {
+                            lostLease(lease, name);
+                        }
+                    },
+                });
+            }
+        } catch (err) {
+            error = err;
+        } finally {
+            lease.close();
+        }
+
+        const outcome = runOutcome(signal, error);
+        if (outcome === 'success') {
+            await write('hdel', 'failures');
+        } else if (outcome === 'failure') {
+            const failures = await write('hincrby', 'failures');
+            if (failures < 0) return;
+            await write(
+                'pexpire',
+                Math.min(job.every, deliveryBackoffMs(failures)),
+            );
+            this.reportFailure(name, failures, error);
+        } else {
+            await write('del');
+        }
+    }
+
+    /**
+     * The lock lapses `leaseMs` after `from`; the run needs half of it left to
+     * start.
+     */
+    #regionLease(leaseMs: number, from: number): RunLease {
+        const controller = new AbortController();
+        const deadline = from + leaseMs;
+        const abort = (reason: string) => {
+            if (!controller.signal.aborted) {
+                controller.abort(new JobLeaseLostError(reason));
+            }
+        };
+        const lapse = setTimeout(
+            () => abort('region lock lapsed'),
+            Math.max(0, deadline - performance.now()),
+        );
+        const lease: RunLease = {
+            controller,
+            ready: async () => {
+                if (
+                    !controller.signal.aborted &&
+                    deadline - performance.now() >= leaseMs / 2
+                ) {
+                    return true;
+                }
+                abort('too little of the lock left to start');
+                return false;
+            },
+            close: () => {
+                clearTimeout(lapse);
+                this.closeLease(lease);
+            },
+        };
+        this.openLease(lease);
+        return lease;
+    }
+
+    /**
+     * Run a write to the region keys if the gate still holds `token`; -1 if
+     * not.
+     */
+    async #regionWrite(
+        keys: RegionKeys,
+        token: string,
+        op: string,
+        ...args: (string | number)[]
+    ): Promise<number> {
+        const result = await this.clients.redis.eval(
+            REGION_FENCED_WRITE,
+            2,
+            keys.gate,
+            keys.run,
+            token,
+            op,
+            ...args,
+        );
+        return Number(result);
+    }
+
+    // -- Hooks, also for extensions that replace `services.jobs` -------
+
+    /**
+     * Run `fleet` jobs registered here and not running in this process. The
+     * default holds each per region, its lease capped at `every`.
+     */
+    protected async pollFleet(names: string[]): Promise<void> {
+        await Promise.all(
+            names.map((name) => {
+                const job = this.definitionOf(name);
+                return this.track(name, () =>
+                    this.#tryRegion(name, {
+                        ...job,
+                        leaseMs: Math.min(job.leaseMs, job.every),
+                    }),
+                );
+            }),
+        );
+    }
+
+    /** Runs `work` as `name`'s only run in this process. */
+    protected track(name: string, work: () => Promise<void>): Promise<void> {
+        const run = (async () => {
+            try {
+                await work();
+            } catch (err) {
+                console.error(`[jobs] ${name} errored`, err);
+            } finally {
+                this.#running.delete(name);
+            }
+        })();
+        this.#running.set(name, run);
+        return run;
+    }
+
+    protected isRunning(name: string): boolean {
+        return this.#running.has(name);
+    }
+
+    protected definitionOf(name: string): RegisteredJob {
+        const job = this.#jobs.get(name);
+        if (!job) throw new Error(`job not registered: ${name}`);
+        return job;
+    }
+
+    /** Who runs a job here: region, server and process. */
+    protected owner(): string {
+        const region = this.services.broadcast.regionId;
+        const server = this.config.serverId ?? 'local';
+        return `${region}:${server}:${process.pid}`.slice(0, 128);
+    }
+
+    /**
+     * Track `lease` for shutdown; one opened while shutting down is aborted at
+     * once.
+     */
+    protected openLease(lease: RunLease): void {
+        this.#leases.add(lease);
+        if (this.#stopping) {
+            lease.controller.abort(
+                new JobLeaseLostError('server shutting down'),
+            );
+        }
+    }
+
+    /** Stop tracking `lease`; its `close` calls this. */
+    protected closeLease(lease: RunLease): void {
+        this.#leases.delete(lease);
+    }
+
+    protected reportFailure(
+        name: string,
+        failures: number,
+        error: unknown,
+    ): void {
+        console.error(`[jobs] ${name} failed (${failures} in a row)`, error);
+        if (failures < JOB_FAILING_ALARM_AT) return;
+        this.clients.alarm.create(
+            `scheduled_job_failing:${this.namespace}:${name}`,
+            `System job ${name} keeps failing`,
+            {
+                namespace: this.namespace,
+                name,
+                failures,
+                error: asError(error),
+            },
+            'warning',
+            { dedup: true },
+        );
+    }
+}
+
+/**
+ * One hash slot per job. Neither part may contain `:` (`JOB_ID_PATTERN`), so no
+ * two (namespace, name) pairs share keys.
+ */
+const regionKeys = (namespace: string, name: string): RegionKeys => {
+    const tag = `{${namespace}:${name}}`;
+    return {
+        lock: `jobs:${tag}:lock`,
+        gate: `jobs:${tag}:next`,
+        run: `jobs:${tag}:run`,
+    };
+};

src/backend/services/jobs/jobRun.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// What one job run needs, whichever scheduler holds the job. Also used by
+// extensions that replace `services.jobs`.
+
+import type { JobDefinition } from './JobScheduler.js';
+
+export const JOB_STATE_MAX_BYTES = 64 * 1024;
+export const JOB_ERROR_MAX_CHARS = 1024;
+
+/** The run no longer holds its job; another node may already be running it. */
+export class JobLeaseLostError extends Error {}
+
+export type RegisteredJob = Required<JobDefinition>;
+
+export type RunOutcome = 'success' | 'failure' | 'interrupted';
+
+/** What a run holds the job by. Its deadline is measured from before the claim. */
+export interface RunLease {
+    controller: AbortController;
+    /**
+     * Renews if little lease is left; false (and aborted) if the run can't
+     * start.
+     */
+    ready: () => Promise<boolean>;
+    close: () => void;
+}
+
+export const lostLease = (lease: RunLease, name: string): never => {
+    const lost = new JobLeaseLostError(`job ${name} lost its lease`);
+    lease.controller.abort(lost);
+    throw lost;
+};
+
+/** A run cut short by its signal is interrupted, whether or not it threw. */
+export const runOutcome = (signal: AbortSignal, error: unknown): RunOutcome => {
+    if (signal.aborted) return 'interrupted';
+    return error === null ? 'success' : 'failure';
+};
+
+export const parseState = (raw: string | null): unknown => {
+    if (raw === null) return null;
+    try {
+        return JSON.parse(raw);
+    } catch {
+        return null;
+    }
+};
+
+export const serializeState = (state: unknown): string => {
+    const json = JSON.stringify(state ?? null);
+    if (Buffer.byteLength(json) > JOB_STATE_MAX_BYTES) {
+        throw new Error(`job state is over ${JOB_STATE_MAX_BYTES} bytes`);
+    }
+    return json;
+};
+
+export const asError = (error: unknown): Error =>
+    error instanceof Error ? error : new Error(String(error));
+
+export const errorText = (error: unknown): string =>
+    (error instanceof Error ? error.message : String(error)).slice(
+        0,
+        JOB_ERROR_MAX_CHARS,
+    );

src/backend/services/jobs/jobsTestUtil.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type { JobRunContext } from './JobScheduler.js';
+
+/**
+ * What a system job's `run` receives, for calling it directly: `saved` records
+ * each `saveState`, and `abort` aborts the signal.
+ */
+export const testJobContext = (
+    state: unknown = null,
+): JobRunContext & { saved: unknown[]; abort: () => void } => {
+    const controller = new AbortController();
+    const saved: unknown[] = [];
+    return {
+        signal: controller.signal,
+        state,
+        saveState: async (next) => {
+            saved.push(next);
+        },
+        saved,
+        abort: () => controller.abort(),
+    };
+};

src/backend/services/jobs/systemJobs.test.ts

@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * The backend's own periodic jobs on the default scheduler: what registers,
+ * that each runs once across nodes sharing Redis, and that a single self-hosted
+ * node (sqlite, in-process Redis) runs every one of them.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import {
+    JobScheduler,
+    type JobDefinition,
+    type JobRunContext,
+} from './JobScheduler.js';
+
+type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
+
+const MINUTE = 60_000;
+const HOUR = 60 * MINUTE;
+
+const EXPECTED_JOBS = {
+    'auth.sessions.purge': {
+        every: 7 * 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 30 * MINUTE,
+    },
+    'kvShare.handles.purgeRevoked': {
+        every: 24 * HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'notifications.retention': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'metering.globalRateCheck': {
+        every: 25 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 5 * MINUTE,
+    },
+    'events.subscriptions.expirySweep': {
+        every: HOUR,
+        scope: 'fleet',
+        nearPrimary: true,
+        leaseMs: 15 * MINUTE,
+    },
+    'events.subscriptions.creditSweep': {
+        every: 15 * MINUTE,
+        scope: 'fleet',
+        nearPrimary: false,
+        leaseMs: 10 * MINUTE,
+    },
+};
+const JOB_NAMES = Object.keys(EXPECTED_JOBS);
+
+/** A timer armed while the server booted, and the frame that armed it. */
+interface ArmedTimer {
+    kind: 'interval' | 'timeout';
+    ms: number | undefined;
+    caller: string;
+}
+
+let server: PuterServer;
+const registered = new Map<string, JobDefinition>();
+const armed: ArmedTimer[] = [];
+
+/** The first stack frame outside this file and node_modules. */
+const callerOf = (stack: string | undefined): string =>
+    (stack ?? '')
+        .split('\n')
+        .slice(1)
+        .find(
+            (line) =>
+                !line.includes('systemJobs.test') &&
+                !line.includes('node_modules'),
+        ) ?? '';
+
+beforeAll(async () => {
+    const register = vi.spyOn(JobScheduler.prototype, 'register');
+    const realInterval = globalThis.setInterval;
+    const realTimeout = globalThis.setTimeout;
+    const intervals = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
+        ...args: Parameters<typeof setInterval>
+    ) => {
+        armed.push({
+            kind: 'interval',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realInterval(...args);
+    }) as never);
+    const timeouts = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+        ...args: Parameters<typeof setTimeout>
+    ) => {
+        armed.push({
+            kind: 'timeout',
+            ms: args[1],
+            caller: callerOf(new Error().stack),
+        });
+        return realTimeout(...args);
+    }) as never);
+    try {
+        server = await setupTestServer({
+            events: { enabled: true },
+            notificationRetentionDays: 14,
+            maxGlobalUsagePerMinute: 1e15,
+            // A self-hosted node: nothing turns its jobs off.
+            jobs: { enabled: true },
+        } as IConfig);
+    } finally {
+        intervals.mockRestore();
+        timeouts.mockRestore();
+    }
+    for (const [name, definition] of register.mock.calls)
+        registered.set(name, definition);
+    register.mockRestore();
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+const keysOf = (namespace: string, name: string) => ({
+    lock: `jobs:{${namespace}:${name}}:lock`,
+    gate: `jobs:{${namespace}:${name}}:next`,
+    run: `jobs:{${namespace}:${name}}:run`,
+});
+
+/** A node of its own in `region`, on the test server's Redis. */
+const node = (namespace: string, region: string): JobScheduler =>
+    new JobScheduler(
+        {
+            domain: 'puter.test',
+            serverId: `node-${region}`,
+            jobs: { namespace },
+        } as IConfig,
+        server.clients as SchedulerArgs[1],
+        server.stores as SchedulerArgs[2],
+        {
+            broadcast: { regionId: region },
+        } as unknown as SchedulerArgs[3],
+    );
+
+describe('system jobs', () => {
+    it('registers every moved job with its real cadence, scope and lease', () => {
+        expect(
+            Object.fromEntries(
+                [...registered].map(([name, job]) => [
+                    name,
+                    {
+                        every: job.every,
+                        scope: job.scope,
+                        nearPrimary: job.nearPrimary ?? false,
+                        leaseMs: job.leaseMs,
+                    },
+                ]),
+            ),
+        ).toEqual(EXPECTED_JOBS);
+    });
+
+    it('leaves no per-node timer behind for a moved job', () => {
+        const armedFrom = (file: string) =>
+            armed
+                .filter(({ caller }) => caller.includes(file))
+                .map(({ kind, ms }) => ({ kind, ms }));
+
+        expect(armedFrom('services/auth/AuthService')).toEqual([]);
+        expect(armedFrom('services/notification/NotificationService')).toEqual(
+            [],
+        );
+        // Node-local: the usage buffer flush.
+        expect(armedFrom('services/metering/MeteringService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+        // Node-local: the share-job runner (DB claims per job), and its kick.
+        expect(armedFrom('services/kvShare/KvShareService')).toEqual([
+            { kind: 'timeout', ms: expect.any(Number) },
+            { kind: 'interval', ms: MINUTE },
+        ]);
+        // Node-local: the region's pending-delivery sweep.
+        expect(armedFrom('services/events/EventsService')).toEqual([
+            { kind: 'interval', ms: 10_000 },
+        ]);
+    });
+
+    it.each(JOB_NAMES)(
+        'runs %s once across three nodes sharing Redis',
+        async (name) => {
+            const namespace = `ns-${randomUUID()}`;
+            const nodes = ['n1', 'n2', 'n3'].map((id) => node(namespace, id));
+            const job = registered.get(name)!;
+            let runs = 0;
+            for (const scheduler of nodes)
+                scheduler.register(name, {
+                    ...job,
+                    run: async (ctx: JobRunContext) => {
+                        runs += 1;
+                        await job.run(ctx);
+                    },
+                });
+
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+            await Promise.all(nodes.map((scheduler) => scheduler.poll()));
+
+            expect(runs).toBe(1);
+            const redis = server.clients.redis;
+            const keys = keysOf(namespace, name);
+            // A failure would have cut the gate to a 2 s backoff.
+            expect(await redis.pttl(keys.gate)).toBeGreaterThan(
+                job.every - MINUTE,
+            );
+            expect(await redis.hget(keys.run, 'failures')).toBeNull();
+        },
+    );
+
+    it('runs every job on a single self-hosted node, without the old per-job gates', async () => {
+        const jobs = server.services.jobs;
+        const redis = server.clients.redis;
+
+        await jobs.poll();
+
+        // The node's own poll timer may have started some of them first.
+        await vi.waitFor(
+            async () => {
+                for (const name of JOB_NAMES) {
+                    const keys = keysOf(jobs.namespace, name);
+                    expect(await redis.exists(keys.gate), name).toBe(1);
+                    // Released once the run is over.
+                    expect(await redis.exists(keys.lock), name).toBe(0);
+                    expect(await redis.hget(keys.run, 'failures'), name).toBe(
+                        null,
+                    );
+                }
+            },
+            { timeout: 10_000, interval: 100 },
+        );
+        await expect(
+            redis.exists('sessions:purge:gate', 'kvShare:purge:gate'),
+        ).resolves.toBe(0);
+    });
+});

src/backend/services/kvShare/KvShareService.ts

@@ -371,11 +371,9 @@ const SHARE_CHANGE_LOCK_RETRY_MS = 50;
 
 /** Retired handles are kept this long for their owner's listing, then deleted. */
 export const KV_SHARE_REVOKED_RETENTION_SECONDS = 30 * 24 * 60 * 60;
-// Every node checks hourly; the gate lets one node through per day.
-const PURGE_CHECK_MS = 60 * 60 * 1000;
-const PURGE_INITIAL_DELAY_MS = 5 * 60 * 1000;
-const PURGE_GATE_KEY = 'kvShare:purge:gate';
-const PURGE_GATE_SECONDS = 24 * 60 * 60;
+const PURGE_JOB = 'kvShare.handles.purgeRevoked';
+const PURGE_EVERY_MS = 24 * 60 * 60 * 1000;
+const PURGE_LEASE_MS = 15 * 60 * 1000;
 // Rows per delete, with a pause between, so replicas keep up.
 const PURGE_BATCH = 500;
 const PURGE_PAUSE_MS = 50;
@@ -536,7 +534,17 @@ export class KvShareService extends PuterService {
     #jobCursor: string | undefined;
 
     override onServerStart(): void {
-        this.#armPurge();
+        this.services.jobs.register(PURGE_JOB, {
+            every: PURGE_EVERY_MS,
+            scope: 'fleet',
+            nearPrimary: true,
+            leaseMs: PURGE_LEASE_MS,
+            run: async ({ signal }) => {
+                const purged = await this.purgeRevokedHandles(signal);
+                if (purged)
+                    console.log(`[kvShare] purged ${purged} retired handles`);
+            },
+        });
         this.#armJobs();
 
         // A check at one level is answered by a grant at any level above it.
@@ -607,51 +615,24 @@ export class KvShareService extends PuterService {
     // -- Retention purge ---------------------------------------------
 
     /**
-     * Delete handles retired longer ago than the retention, in batches. Returns
-     * the rows deleted, or null when another node holds today's gate. A missing
-     * row reads like a retired one everywhere a handle is resolved.
+     * Delete handles retired longer ago than the retention, in batches, until
+     * none are left or `signal` aborts. Returns the rows deleted. A missing row
+     * reads like a retired one everywhere a handle is resolved.
      */
-    async purgeRevokedHandles(): Promise<number | null> {
-        const claimed = await this.clients.redis.set(
-            PURGE_GATE_KEY,
-            String(Date.now()),
-            'EX',
-            PURGE_GATE_SECONDS,
-            'NX',
-        );
-        if (claimed !== 'OK') return null;
-
+    async purgeRevokedHandles(signal: AbortSignal): Promise<number> {
         const cutoff =
             Math.floor(Date.now() / 1000) - KV_SHARE_REVOKED_RETENTION_SECONDS;
         let purged = 0;
-        let finished = false;
-        try {
-            for (let i = 0; i < PURGE_MAX_BATCHES; i++) {
-                if (this.#stopping) return purged;
-                const batch =
-                    await this.stores.kvShareHandle.deleteRevokedBefore(
-                        cutoff,
-                        PURGE_BATCH,
-                    );
-                purged += batch;
-                if (batch < PURGE_BATCH) break;
-                await new Promise((resolve) =>
-                    setTimeout(resolve, PURGE_PAUSE_MS),
-                );
-            }
-            finished = true;
-            return purged;
-        } finally {
-            // A run cut short frees the gate, so another node picks it up on
-            // its next check instead of tomorrow.
-            if (!finished) {
-                try {
-                    await this.clients.redis.del(PURGE_GATE_KEY);
-                } catch {
-                    // The gate expires on its own.
-                }
-            }
+        for (let i = 0; i < PURGE_MAX_BATCHES && !signal.aborted; i++) {
+            const batch = await this.stores.kvShareHandle.deleteRevokedBefore(
+                cutoff,
+                PURGE_BATCH,
+            );
+            purged += batch;
+            if (batch < PURGE_BATCH) break;
+            await new Promise((resolve) => setTimeout(resolve, PURGE_PAUSE_MS));
         }
+        return purged;
     }
 
     /**
@@ -1047,27 +1028,6 @@ export class KvShareService extends PuterService {
         this.#timers.push(kick, timer);
     }
 
-    #armPurge(): void {
-        const run = async () => {
-            try {
-                const purged = await this.purgeRevokedHandles();
-                if (purged)
-                    console.log(`[kvShare] purged ${purged} retired handles`);
-            } catch (err) {
-                console.warn('[kvShare] retired-handle purge failed', err);
-            }
-        };
-        // Jittered so a deploy does not have every node check at once.
-        const kick = setTimeout(
-            () => void run(),
-            PURGE_INITIAL_DELAY_MS * (0.5 + Math.random()),
-        );
-        kick.unref?.();
-        const timer = setInterval(() => void run(), PURGE_CHECK_MS);
-        timer.unref?.();
-        this.#timers.push(kick, timer);
-    }
-
     /** Whether the level a mint asks for is available on this server. */
     #assertAccessEnabled(access: KvShareAccess): void {
         if (access === 'watch') {
@@ -1745,11 +1705,14 @@ export class KvShareService extends PuterService {
      * The other party of a share, for a notification one party sends through
      * its handle from the app whose data it is. Every check runs before the
      * pair budget is spent. A handle that is unknown, retired, not the
-     * caller's, or seen from another app reads as absent.
+     * caller's, or seen from another app reads as absent. `spendBudget: false`
+     * checks without spending, for a send that is only being scheduled. Called
+     * by extensions.
      */
     async resolveNotifyRecipient(
         actor: Actor,
         handle: unknown,
+        { spendBudget = true }: { spendBudget?: boolean } = {},
     ): Promise<KvShareNotifyTarget> {
         const named = typeof handle === 'string' ? handle : '';
         const senderId = actor.user?.id;
@@ -1782,7 +1745,8 @@ export class KvShareService extends PuterService {
         if (!checked?.holds) throw unknownKvShareHandle(named);
         if (checked.refuses) throw recipientNotAcceptingShares();
 
-        await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
+        if (spendBudget)
+            await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
         return {
             recipientUserId: recipientId,
             appUid: share.appUid,

src/backend/services/metering/MeteringService.ts

@@ -111,6 +111,11 @@ type UsageWriteResult = {
     detail?: FlatUsageDetail;
 };
 
+/** Compares fleet-wide spend against `maxGlobalUsagePerMinute`. */
+const GLOBAL_RATE_CHECK_JOB = 'metering.globalRateCheck';
+const GLOBAL_RATE_CHECK_EVERY_MS = 25 * 60 * 1000;
+const GLOBAL_RATE_CHECK_LEASE_MS = 5 * 60 * 1000;
+
 // -- Helpers ----------------------------------------------------------
 
 /**
@@ -284,7 +289,6 @@ export class MeteringService extends PuterService {
      */
     static APP_TOTALS_LIST_LIMIT = 4_000;
 
-    private rateCheckTimer: ReturnType<typeof setInterval> | null = null;
     private usageBufferTimer: ReturnType<typeof setInterval> | null = null;
     private extraPolicies: SubscriptionPolicy[] = [];
     private subscriptionResolvers: SubscriptionResolver[] = [];
@@ -415,15 +419,15 @@ export class MeteringService extends PuterService {
             },
         );
 
-        this.rateCheckTimer = setInterval(
-            () => {
-                this.checkRateOfChange().catch((e) => {
-                    console.error('[metering] rate-of-change check failed', e);
-                });
-            },
-            1000 * 60 * 25,
-        );
-        this.rateCheckTimer.unref?.();
+        const maxPerMinute = this.config.maxGlobalUsagePerMinute;
+        if (maxPerMinute && maxPerMinute > 0) {
+            this.services.jobs.register(GLOBAL_RATE_CHECK_JOB, {
+                every: GLOBAL_RATE_CHECK_EVERY_MS,
+                scope: 'fleet',
+                leaseMs: GLOBAL_RATE_CHECK_LEASE_MS,
+                run: () => this.checkRateOfChange(),
+            });
+        }
 
         const flushInterval =
             this.config.meteringUsageBufferFlushMs &&
@@ -454,10 +458,6 @@ export class MeteringService extends PuterService {
     }
 
     override async onServerShutdown(): Promise<void> {
-        if (this.rateCheckTimer) {
-            clearInterval(this.rateCheckTimer);
-            this.rateCheckTimer = null;
-        }
         if (this.usageBufferTimer) {
             clearInterval(this.usageBufferTimer);
             this.usageBufferTimer = null;
@@ -2948,6 +2948,8 @@ export class MeteringService extends PuterService {
             timestamp: number;
         } | null;
 
+        // Several schedulers can share the baseline (one per region without a
+        // fleet scheduler, or old nodes mid-rollout); keep windows wide.
         if (lastChange && now - lastChange.timestamp <= 14 * 60 * 1000) return;
 
         const globalUsage = await this.getGlobalUsage();

src/backend/services/notification/NotificationService.test.ts

@@ -395,6 +395,36 @@ describe('NotificationService.notifyUpdate', () => {
         written.stop();
     });
 
+    it('sends the desktop rewrite before reading the row back, so a failed read still updates it', async () => {
+        const user = await makeUser();
+        const uid = await notifications.notify(
+            [user.id],
+            { title: 'one share' },
+            { type: 'share.received' },
+        );
+        const pushed = collect('outer.gui.notif.message');
+        const read = vi
+            .spyOn(server.stores.notification, 'getByUid')
+            .mockRejectedValueOnce(new Error('replica down'));
+        try {
+            await expect(
+                notifications.notifyUpdate(
+                    uid,
+                    user.id,
+                    { title: 'two shares' },
+                    { type: 'share.received' },
+                ),
+            ).rejects.toThrow('replica down');
+            expect(pushed.seen).toHaveLength(1);
+            expect((pushed.seen[0] as NotifPush).response.notification).toEqual(
+                { title: 'two shares', type: 'share.received' },
+            );
+        } finally {
+            read.mockRestore();
+            pushed.stop();
+        }
+    });
+
     it('refuses to fold a type that does not group', async () => {
         const user = await makeUser();
         await expect(
@@ -882,17 +912,42 @@ describe('NotificationService.sweepExpired', () => {
     });
 
     it('stops at the pass cap and leaves the rest for the next sweep', async () => {
+        // A backlog that always fills the batch: the cap, not a short batch,
+        // is what ends the call. Seeding 500k rows would take minutes.
+        const deletes = vi
+            .spyOn(server.stores.notification, 'deleteCreatedBefore')
+            .mockResolvedValue(500);
+        try {
+            expect(await notifications.sweepExpired()).toBe(500_000);
+            expect(deletes).toHaveBeenCalledTimes(1000);
+        } finally {
+            deletes.mockRestore();
+        }
+    });
+
+    it('stops between batches once its signal aborts', async () => {
         await notifications.sweepExpired();
         const user = await makeUser();
-        // 50 passes * 500/batch = 25,000 — a backlog past that so the cap,
-        // not a short batch, is what ends the first call.
-        await seedExpired(user.id, 25_050);
-
-        expect(await notifications.sweepExpired()).toBe(25_000);
-        expect(
-            await server.stores.notification.listByUserId(user.id),
-        ).toHaveLength(50);
-        expect(await notifications.sweepExpired()).toBe(50);
+        await seedExpired(user.id, 1200);
+        const controller = new AbortController();
+        const deleteBatch = server.stores.notification.deleteCreatedBefore.bind(
+            server.stores.notification,
+        );
+        const spy = vi
+            .spyOn(server.stores.notification, 'deleteCreatedBefore')
+            .mockImplementation(async (days: number, limit: number) => {
+                const removed = await deleteBatch(days, limit);
+                controller.abort();
+                return removed;
+            });
+        try {
+            expect(await notifications.sweepExpired(controller.signal)).toBe(
+                500,
+            );
+        } finally {
+            spy.mockRestore();
+        }
+        expect(await notifications.sweepExpired()).toBe(700);
     });
 
     it('sweeps nothing when no retention is configured', async () => {

src/backend/services/notification/NotificationService.ts

@@ -48,12 +48,16 @@ export type {
 } from './notificationTypes.js';
 export { canViewNotification } from './notificationAudience.js';
 
-/** How often the retention sweep runs. */
-const RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+const RETENTION_SWEEP_JOB = 'notifications.retention';
+const RETENTION_SWEEP_EVERY_MS = 60 * 60 * 1000;
+const RETENTION_SWEEP_LEASE_MS = 15 * 60 * 1000;
 /** Rows one delete takes. Small enough not to hold a lock anyone waits on. */
 const RETENTION_BATCH_SIZE = 500;
-/** Batches one sweep takes, so a large backlog drains over several passes. */
-const RETENTION_MAX_BATCHES = 50;
+/**
+ * Batches one sweep takes at most. It runs once for the fleet, so this is the
+ * fleet's hourly drain; the job's signal stops it sooner.
+ */
+const RETENTION_MAX_BATCHES = 1000;
 /** Unread notifications one reconnect replays. */
 const UNREAD_REPLAY_LIMIT = 200;
 
@@ -69,11 +73,20 @@ const UNREAD_REPLAY_LIMIT = 200;
 export class NotificationService extends PuterService {
     /** User.id → debounce timeout */
     #connectTimeouts = new Map<number, ReturnType<typeof setTimeout>>();
-    #retentionSweep: ReturnType<typeof setInterval> | null = null;
     #adapter: NotificationSocketAdapter | null = null;
 
     override onServerStart(): void {
-        this.#armRetentionSweep();
+        if (this.#retentionDays() > 0) {
+            this.services.jobs.register(RETENTION_SWEEP_JOB, {
+                every: RETENTION_SWEEP_EVERY_MS,
+                scope: 'fleet',
+                nearPrimary: true,
+                leaseMs: RETENTION_SWEEP_LEASE_MS,
+                run: async ({ signal }) => {
+                    await this.sweepExpired(signal);
+                },
+            });
+        }
         this.#socket().attach();
 
         // When a user opens the GUI, send their pending unreads.
@@ -103,11 +116,6 @@ export class NotificationService extends PuterService {
         );
     }
 
-    override onServerPrepareShutdown(): void {
-        if (this.#retentionSweep) clearInterval(this.#retentionSweep);
-        this.#retentionSweep = null;
-    }
-
     // -- Public API --------------------------------------------------
 
     /**
@@ -176,25 +184,17 @@ export class NotificationService extends PuterService {
                 }
 
                 if (!opts.silent) {
-                    // Listeners get their own copy; the socket sends `payload`.
-                    this.clients.event.emit(
-                        'notif.written',
+                    await this.emitWritten(
                         {
                             userId,
                             uid,
                             type: registered.type,
                             audience: registered.audience,
                             appUid,
-                            value: structuredClone(payload),
-                            sendId: uuidv4(),
+                            value: payload,
                         },
-                        {},
+                        { sendId: uuidv4() },
                     );
-                    await this.#push(userId, uid, payload, {
-                        type: registered.type,
-                        audience: registered.audience,
-                        appUid,
-                    });
                 }
                 return uid;
             }),
@@ -241,50 +241,107 @@ export class NotificationService extends PuterService {
         if (!updated) return false;
         if (opts.silent) return true;
 
+        // Before the read, so the desktop copy updates even if it fails.
         const desktop = inAccountMailbox(registered.type);
-        if (desktop) {
-            // Same uid as the original: the client replaces what it is already
-            // showing rather than stacking another copy. It goes straight to
-            // the wire rather than through dispatch — the row was already
-            // published, and republishing it under its own id is a duplicate
-            // to anything deduplicating on `event.id`.
-            this.#socket().message(userId, uid, payload);
-            this.#markDelivered(uid, userId);
-        }
+        if (desktop) this.#sendRewrite(userId, uid, payload);
         const row = await this.stores.notification.getByUid(uid, { userId });
         const app = notification.app as { uid?: unknown } | undefined;
         const appUid =
             row?.app_uid ?? (typeof app?.uid === 'string' ? app.uid : null);
-        this.clients.event.emit(
-            'notif.written',
+        await this.emitWritten(
             {
                 userId,
                 uid,
                 type: registered.type,
                 audience: registered.audience,
                 appUid,
-                value: structuredClone(payload),
+                value: payload,
                 updated: true,
-                sendId: uuidv4(),
+                replacedAt,
+            },
+            { sendId: uuidv4(), rewriteSent: desktop },
+        );
+        return true;
+    }
+
+    /**
+     * Deliver a notification already written: `notif.written` for listeners,
+     * then the live copy. `sendId` names this one send; a delayed send passes
+     * the same one each time it delivers, so a listener can tell a repeat. An
+     * `updated` row is an earlier notification rewritten in place, and
+     * `replacedAt` marks it a replacement. `handlerDepth` is the depth the
+     * write was made at, when no actor in context carries it. `rewriteSent`
+     * says the caller already sent a desktop rewrite. `replayable` marks a send
+     * that may be emitted again under the same `sendId`. Called by extensions.
+     */
+    async emitWritten(
+        written: {
+            userId: number;
+            uid: string;
+            type: string;
+            audience: string;
+            appUid: string | null;
+            value: Record<string, unknown>;
+            updated?: boolean;
+            replacedAt?: number;
+        },
+        opts: {
+            sendId: string;
+            handlerDepth?: number;
+            rewriteSent?: boolean;
+            replayable?: boolean;
+        },
+    ): Promise<void> {
+        const { userId, uid, value: payload, updated } = written;
+        const scope = {
+            type: written.type,
+            audience: written.audience,
+            appUid: written.appUid,
+        };
+        const desktop = inAccountMailbox(scope.type);
+        if (updated && desktop && !opts.rewriteSent)
+            this.#sendRewrite(userId, uid, payload);
+        // Listeners get their own copy; the socket sends `payload`.
+        this.clients.event.emit(
+            'notif.written',
+            {
+                userId,
+                uid,
+                ...scope,
+                value: structuredClone(payload),
+                ...(updated ? { updated: true as const } : {}),
+                sendId: opts.sendId,
+                ...(opts.replayable ? { replayable: true as const } : {}),
             },
             {},
         );
-        // A type kept off the desktop has dispatch as its only live path, so
-        // the rewrite goes out there again, under the same uid.
-        if (!desktop) {
+        // A rewrite of a type kept off the desktop has dispatch as its only
+        // live path, so it goes out there again, under the same uid.
+        if (!updated || !desktop)
             await this.#push(
                 userId,
                 uid,
                 payload,
-                {
-                    type: registered.type,
-                    audience: registered.audience,
-                    appUid,
-                },
-                replacedAt,
+                scope,
+                written.replacedAt,
+                opts.handlerDepth,
             );
-        }
-        return true;
+    }
+
+    /**
+     * Same uid as the original: the client replaces what it is already showing
+     * rather than stacking another copy. It goes straight to the wire rather
+     * than through dispatch — the row was already published, and republishing
+     * it under its own id is a duplicate to anything deduplicating on
+     * `event.id`.
+     */
+    #sendRewrite(
+        userId: number,
+        uid: string,
+        payload: Record<string, unknown>,
+    ): void {
+        this.#socket().message(userId, uid, payload);
+        this.#markDelivered(uid, userId);
     }
 
     /**
@@ -348,17 +405,15 @@ export class NotificationService extends PuterService {
      * Drop notifications past the retention window, in batches, and report how
      * many went. Deleting is all there is to it: nothing is pushed, because a
      * two-week-old row is not news, and a client listing again simply stops
-     * seeing it.
-     *
-     * Every node sweeps. Batches are small and the delete is idempotent, so two
-     * nodes overlapping costs a few empty batches, not correctness.
+     * seeing it. Stops between batches once `signal` aborts.
      */
-    async sweepExpired(): Promise<number> {
+    async sweepExpired(signal?: AbortSignal): Promise<number> {
         const days = this.#retentionDays();
         if (days <= 0) return 0;
 
         let removed = 0;
         for (let pass = 0; pass < RETENTION_MAX_BATCHES; pass++) {
+            if (signal?.aborted) break;
             const batch = await this.stores.notification.deleteCreatedBefore(
                 days,
                 RETENTION_BATCH_SIZE,
@@ -411,6 +466,7 @@ export class NotificationService extends PuterService {
         payload: Record<string, unknown>,
         scope: { type: string; audience: string; appUid: string | null },
         replacedAt?: number,
+        depth?: number,
     ): Promise<void> {
         const desktop = inAccountMailbox(scope.type);
         if (!notificationsFoldInEnabled(this.config)) {
@@ -430,9 +486,8 @@ export class NotificationService extends PuterService {
             return;
         }
 
-        const handlerDepth = handlerDepthOf(
-            Context.get('actor') as Actor | undefined,
-        );
+        const handlerDepth =
+            depth ?? handlerDepthOf(Context.get('actor') as Actor | undefined);
         this.clients.event.emit(
             'notif.created',
             {
@@ -470,17 +525,6 @@ export class NotificationService extends PuterService {
         return Number.isFinite(configured) && configured > 0 ? configured : 0;
     }
 
-    #armRetentionSweep(): void {
-        if (this.#retentionDays() <= 0) return;
-        const sweep = setInterval(() => {
-            void this.sweepExpired().catch((err) => {
-                console.warn('[notification] retention sweep failed', err);
-            });
-        }, RETENTION_SWEEP_INTERVAL_MS);
-        sweep.unref?.();
-        this.#retentionSweep = sweep;
-    }
-
     /**
      * What a reconnecting client missed. One statement marks the batch shown —
      * a per-row update is a round trip per notification on a path that runs

src/backend/services/types.ts

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import type { ScheduleServiceContract } from '../core/schedules.js';
 import type { TeamServiceContract } from '../core/teams.js';
 
 import type { puterClients } from '../clients';
@@ -38,6 +39,8 @@ export interface IPuterServiceInstances {}
 export interface IExtensionServiceInstances {
     /** Teams ships separately; boot guarantees it when `teams_enabled` is on. */
     team?: TeamServiceContract;
+    /** Schedules ship separately; without them their routes answer 503. */
+    schedules?: ScheduleServiceContract;
     [key: string]: unknown;
 }
 

src/backend/stores/app/AppStore.js

@@ -262,6 +262,19 @@ export class AppStore extends PuterStore {
         return result;
     }
 
+    /**
+     * Cache-free existence check by uid against the primary. Called by
+     * extensions.
+     */
+    async uidExistsOnPrimary(uid) {
+        if (!uid) return false;
+        const rows = await this.clients.db.pread(
+            'SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1',
+            [uid],
+        );
+        return rows.length > 0;
+    }
+
     /** Cache-free existence check against the primary. */
     async existsOnPrimary(appId) {
         if (appId === undefined || appId === null) return false;

src/backend/stores/events/DurableSubscriptionStore.integration.test.ts

@@ -43,6 +43,7 @@ import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
 import type { IConfig } from '../../types.js';
 import { EVENT_SUBSCRIPTION_WIDTHS } from './columnWidths.js';
 import type { DurableSubscriptionInput } from './DurableSubscriptionStore.js';
+import { scheduleToken } from './types.js';
 
 const BOOT_TIMEOUT_MS = 120_000;
 
@@ -201,9 +202,9 @@ describe('creating a subscription', () => {
         const { row } = await durable().create(input());
 
         await expect(cache().userHasAny(userId)).resolves.toBe(true);
-        await expect(
-            cache().watchedTokens(userId, [token()]),
-        ).resolves.toEqual([token()]);
+        await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
+            [token()],
+        );
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             row,
         ]);
@@ -468,7 +469,9 @@ describe('the per-account cap', () => {
             input({ expiresAt: Math.floor(Date.now() / 1000) - 60 }),
         );
         await durable().create(input());
-        await expect(durable().countForHolder(userId)).resolves.toMatchObject({ total: 1 });
+        await expect(durable().countForHolder(userId)).resolves.toMatchObject({
+            total: 1,
+        });
     });
 });
 
@@ -547,9 +550,13 @@ describe('rows suspended for good', () => {
         await durable().suspend([failing], 'failures');
         await durable().suspend([revoked], 'permission_revoked');
 
-        const held = await durable().listActiveForHolder(userId, null, {
-            includeSuspended: true,
-        });
+        const { rows: held } = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            0,
+            undefined,
+            { includeSuspended: true },
+        );
         expect(held.map((row) => row.subId).sort()).toEqual(
             [live.subId, failing.subId].sort(),
         );
@@ -565,7 +572,9 @@ describe('rows suspended for good', () => {
             },
         );
 
-        await durable().create(input());
+        const created = await durable().create(input());
+        // The trim's bump, for the caller to publish to the other regions.
+        expect(created.reaped.map((bump) => bump.userId)).toEqual([userId]);
 
         await expect(durable().countForHolder(userId)).resolves.toEqual({
             total: 1,
@@ -600,9 +609,9 @@ describe('removal', () => {
 
         await durable().remove(row);
 
-        await expect(
-            cache().watchedTokens(userId, [token()]),
-        ).resolves.toEqual([token()]);
+        await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
+            [token()],
+        );
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             survivor.row,
         ]);
@@ -744,9 +753,11 @@ describe('the expiry sweep', () => {
         // one back to prove the sweep is what removes it.
         await cache().cacheDurable([expired.row]);
 
-        await expect(durable().sweepExpired(500)).resolves.toBe(1);
+        expect((await durable().sweepExpired(500)).rows).toHaveLength(1);
 
-        await expect(durable().getBySubId(expired.row.subId)).resolves.toBeNull();
+        await expect(
+            durable().getBySubId(expired.row.subId),
+        ).resolves.toBeNull();
         await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
             live.row,
         ]);
@@ -754,19 +765,42 @@ describe('the expiry sweep', () => {
 
     it('leaves a subscription with no expiry alone', async () => {
         await durable().create(input());
-        await expect(durable().sweepExpired(500)).resolves.toBe(0);
+        expect((await durable().sweepExpired(500)).rows).toHaveLength(0);
         await expect(durable().countForHolder(userId)).resolves.toMatchObject({
             total: 1,
         });
     });
 });
 
+describe('a sweep reading a lagging replica', () => {
+    it('keeps a row the primary resumed since, and does not report it', async () => {
+        const { row } = await durable().create(input());
+        await durable().suspend([row], 'permission_revoked');
+        const [stale] = await env.server.clients.db.pread(
+            'SELECT * FROM `event_subscriptions` WHERE `sub_id` = ?',
+            [row.subId],
+        );
+        await durable().resume([row]);
+        // The replica still has it suspended.
+        vi.spyOn(env.server.clients.db, 'read').mockResolvedValueOnce([stale]);
+
+        const reaped = await durable().sweepSuspended(
+            Math.floor(Date.now() / 1000) + 3600,
+            500,
+        );
+
+        expect(reaped.rows).toEqual([]);
+        await expect(durable().getBySubId(row.subId)).resolves.toMatchObject({
+            subId: row.subId,
+            suspendedAt: null,
+        });
+    });
+});
+
 describe('listHolderIdsForApp', () => {
     it('names every distinct holder bound to the app`s handlers', async () => {
         const appUid = `app-${uuidv4()}`;
-        await durable().create(
-            input({ appUid, holderUserId: userId }),
-        );
+        await durable().create(input({ appUid, holderUserId: userId }));
         await durable().create(
             input({
                 appUid,
@@ -774,9 +808,7 @@ describe('listHolderIdsForApp', () => {
             }),
         );
         // A second row for the same holder must not double the name.
-        await durable().create(
-            input({ appUid, holderUserId: userId }),
-        );
+        await durable().create(input({ appUid, holderUserId: userId }));
 
         const holders = await durable().listHolderIdsForApp(appUid);
         expect(holders.sort()).toEqual([userId, otherUserId].sort());
@@ -784,48 +816,44 @@ describe('listHolderIdsForApp', () => {
 
     it('finds a suspended row`s holder too', async () => {
         const appUid = `app-${uuidv4()}`;
-        const { row } = await durable().create(
-            input({ appUid }),
-        );
+        const { row } = await durable().create(input({ appUid }));
         await durable().suspend([row], 'handler_not_found');
 
         expect(await durable().listHolderIdsForApp(appUid)).toEqual([userId]);
     });
 
     it('answers an empty list for an app with nothing bound', async () => {
-        expect(
-            await durable().listHolderIdsForApp(`app-${uuidv4()}`),
-        ).toEqual([]);
+        expect(await durable().listHolderIdsForApp(`app-${uuidv4()}`)).toEqual(
+            [],
+        );
     });
 });
 
 describe('reapForApp', () => {
     it('deletes every row bound to the app and bumps its owner`s generation', async () => {
         const appUid = `app-${uuidv4()}`;
-        const mine = await durable().create(
-            input({ appUid }),
-        );
+        const mine = await durable().create(input({ appUid }));
         const untouched = await durable().create(input());
         const before = await cache().getGeneration(userId);
 
-        await expect(durable().reapForApp(appUid, 500)).resolves.toBe(1);
+        const reaped = await durable().reapForApp(appUid, 500);
+        expect(reaped.rows.map((row) => row.subId)).toEqual([mine.row.subId]);
 
         await expect(durable().getBySubId(mine.row.subId)).resolves.toBeNull();
         await expect(
             durable().getBySubId(untouched.row.subId),
         ).resolves.not.toBeNull();
-        await expect(cache().getGeneration(userId)).resolves.toBeGreaterThan(
-            before,
-        );
+        const after = await cache().getGeneration(userId);
+        expect(after).toBeGreaterThan(before);
+        // The bump the caller publishes to the other regions.
+        expect(reaped.bumps).toEqual([{ userId, generation: after }]);
     });
 
     it('never touches another app`s rows', async () => {
         const appUid = `app-${uuidv4()}`;
         const otherAppUid = `app-${uuidv4()}`;
         await durable().create(input({ appUid }));
-        const theirs = await durable().create(
-            input({ appUid: otherAppUid }),
-        );
+        const theirs = await durable().create(input({ appUid: otherAppUid }));
 
         await durable().reapForApp(appUid, 500);
 
@@ -837,7 +865,7 @@ describe('reapForApp', () => {
     it('answers 0 for an app with nothing to reap', async () => {
         await expect(
             durable().reapForApp(`app-${uuidv4()}`, 500),
-        ).resolves.toBe(0);
+        ).resolves.toEqual({ rows: [], bumps: [] });
     });
 });
 
@@ -904,3 +932,298 @@ describe('suspending', () => {
         expect(second.bumps).toEqual([]);
     });
 });
+
+describe('rows a schedule owns', () => {
+    const scheduleInput = (
+        over: Partial<DurableSubscriptionInput> = {},
+    ): DurableSubscriptionInput => {
+        const uid = uuidv4();
+        return input({
+            subject: scheduleToken(uid),
+            token: scheduleToken(uid),
+            anchorUid: uid,
+            anchorPath: '',
+            appUid: 'app-scheduler',
+            delivery: 'single',
+            handlerName: 'tick',
+            permission: 'events:background',
+            ...over,
+        });
+    };
+
+    it('are created past the durable quota, uncached and with no bump', async () => {
+        const limits = { perUser: 1, perApp: 1 };
+        await durable().create(input({ limits }));
+        const before = await cache().getGeneration(userId);
+
+        const created = await durable().create(scheduleInput({ limits }));
+
+        expect(created.bump).toBeNull();
+        await expect(cache().getGeneration(userId)).resolves.toBe(before);
+        await expect(
+            cache().getForTokens(userId, [created.row.token]),
+        ).resolves.toEqual([]);
+        await expect(
+            durable().getBySubId(created.row.subId),
+        ).resolves.toMatchObject({ token: created.row.token });
+    });
+
+    it('never enter the dispatch cache or what a cold region rebuilds', async () => {
+        const { row } = await durable().create(scheduleInput());
+        await expect(
+            durable().listDeliverableForOwner(userId),
+        ).resolves.toEqual([]);
+
+        await cache().markRegionCold(userId);
+        await durable().warmRegion(userId);
+        await expect(cache().userHasAny(userId)).resolves.toBe(false);
+
+        await durable().suspend([row], 'no_credit');
+        await durable().resume([
+            { ...row, suspendedAt: 1, suspendedReason: 'no_credit' },
+        ]);
+        await expect(
+            cache().getForTokens(userId, [row.token]),
+        ).resolves.toEqual([]);
+        await expect(cache().userHasAny(userId)).resolves.toBe(false);
+    });
+
+    it('bump no generation when suspended or resumed', async () => {
+        const { row } = await durable().create(scheduleInput());
+        const before = await cache().getGeneration(userId);
+
+        const { suspended, bumps } = await durable().suspend([row], 'failures');
+        expect(suspended).toHaveLength(1);
+        expect(bumps).toEqual([]);
+        await expect(durable().resume(suspended)).resolves.toEqual([]);
+        await expect(cache().getGeneration(userId)).resolves.toBe(before);
+    });
+
+    it('count toward no durable quota, and are not in the durable listing', async () => {
+        await durable().create(input());
+        await durable().create(scheduleInput());
+
+        await expect(durable().countForHolder(userId)).resolves.toEqual({
+            total: 1,
+            forApp: 0,
+            terminal: 0,
+        });
+        const listed = await durable().listForHolder(userId, {
+            includeTotal: true,
+        });
+        expect(listed.total).toBe(1);
+        expect(listed.items.map((row) => row.token)).toEqual([token()]);
+    });
+
+    it('are kept when the oldest revoked rows are trimmed', async () => {
+        const base = Math.floor(Date.now() / 1000) - 10_000;
+        const { row: scheduled } = await durable().create(scheduleInput());
+        await env.server.clients.db.write(
+            'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
+                '`suspended_reason` = ? WHERE `sub_id` = ?',
+            [base - 1, 'permission_revoked', scheduled.subId],
+        );
+        for (let i = 0; i < EVENTS_REVOKED_SUBSCRIPTIONS_KEPT + 1; i++) {
+            const { row } = await durable().create(input());
+            await env.server.clients.db.write(
+                'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
+                    '`suspended_reason` = ? WHERE `sub_id` = ?',
+                [base + i, 'permission_revoked', row.subId],
+            );
+        }
+
+        await durable().create(input());
+
+        await expect(
+            durable().getBySubId(scheduled.subId),
+        ).resolves.not.toBeNull();
+        await expect(durable().countForHolder(userId)).resolves.toMatchObject({
+            terminal: EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
+        });
+    });
+
+    it('are read, a page at a time, by what settles a holder', async () => {
+        const made: string[] = [];
+        for (let i = 0; i < 3; i++)
+            made.push((await durable().create(scheduleInput())).row.subId);
+        made.push((await durable().create(input())).row.subId);
+
+        const first = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            0,
+            2,
+        );
+        expect(first.rows.map((row) => row.subId)).toEqual(made.slice(0, 2));
+        expect(first.nextId).not.toBeNull();
+        const second = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            first.nextId!,
+            2,
+        );
+        expect(second.rows.map((row) => row.subId)).toEqual(made.slice(2));
+        // A full page always says to look again; the next one is empty.
+        const third = await durable().listActiveForHolderPage(
+            userId,
+            null,
+            second.nextId ?? Number.MAX_SAFE_INTEGER,
+            2,
+        );
+        expect(third).toEqual({ rows: [], nextId: null });
+    });
+
+    it('are read, a page at a time, by what a top-up resumes', async () => {
+        const rows = [];
+        for (let i = 0; i < 3; i++)
+            rows.push((await durable().create(scheduleInput())).row);
+        await durable().suspend(rows, 'no_credit');
+
+        const first = await durable().listSuspendedForHolderPage(
+            userId,
+            'no_credit',
+            0,
+            2,
+        );
+        expect(first.rows).toHaveLength(2);
+        const second = await durable().listSuspendedForHolderPage(
+            userId,
+            'no_credit',
+            first.nextId!,
+            2,
+        );
+        expect(second.rows.map((row) => row.subId)).toEqual([rows[2].subId]);
+        expect(second.nextId).toBeNull();
+    });
+
+    it('are found by handler and by app, like any other row', async () => {
+        const { row } = await durable().create(scheduleInput());
+        await expect(
+            durable().listByHandler('app-scheduler', 'tick'),
+        ).resolves.toEqual([expect.objectContaining({ subId: row.subId })]);
+        await expect(
+            durable().listHolderIdsForApp('app-scheduler'),
+        ).resolves.toEqual([userId]);
+    });
+
+    it('come back from a reap without touching a backlog', async () => {
+        const { row } = await durable().create(scheduleInput());
+        const regular = await durable().create(
+            input({ appUid: 'app-scheduler' }),
+        );
+        const purge = vi.spyOn(env.server.stores.pendingDelivery, 'purge');
+        try {
+            const reaped = await durable().reapForApp('app-scheduler', 500);
+            expect(reaped.rows.map((r) => r.subId).sort()).toEqual(
+                [row.subId, regular.row.subId].sort(),
+            );
+            expect(purge.mock.calls.map(([subId]) => subId)).toEqual([
+                regular.row.subId,
+            ]);
+            // Only the cached row's owner was bumped.
+            expect(reaped.bumps).toHaveLength(1);
+        } finally {
+            purge.mockRestore();
+        }
+    });
+
+    it('are deleted by their schedule`s id, whether or not it recorded the row', async () => {
+        const uid = uuidv4();
+        await durable().create(
+            scheduleInput({
+                subject: scheduleToken(uid),
+                token: scheduleToken(uid),
+                anchorUid: uid,
+            }),
+        );
+        await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
+            true,
+        );
+        await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
+            false,
+        );
+    });
+});
+
+describe('statements a schedule store composes', () => {
+    const createScheduleRow = async () => {
+        const uid = uuidv4();
+        const { row } = await durable().create(
+            input({
+                subject: scheduleToken(uid),
+                token: scheduleToken(uid),
+                anchorUid: uid,
+                anchorPath: '',
+                appUid: 'app-scheduler',
+                delivery: 'single',
+                handlerName: 'tick',
+                context: '{"region":"eu"}',
+                permission: 'events:background',
+            }),
+        );
+        return { uid, row };
+    };
+
+    it('delete a schedule`s row only while the guard holds', async () => {
+        const { uid, row } = await createScheduleRow();
+        const db = env.server.clients.db;
+
+        await db.batchWrite([
+            durable().scheduleSubscriptionDeleteEntry(uid, {
+                table: 'user',
+                where: '`id` = ?',
+                values: [-1],
+            }),
+        ]);
+        await expect(durable().getBySubId(row.subId)).resolves.not.toBeNull();
+
+        await db.batchWrite([
+            durable().scheduleSubscriptionDeleteEntry(uid, {
+                table: 'user',
+                where: '`id` = ?',
+                values: [userId],
+            }),
+        ]);
+        await expect(durable().getBySubId(row.subId)).resolves.toBeNull();
+    });
+
+    it('join a schedule`s row under the sub_* columns', async () => {
+        const { row } = await createScheduleRow();
+        const join = durable().scheduleSubscriptionJoin('sub', 'x.`sub_id`');
+        const read = (subId: string) =>
+            env.server.clients.db.pread(
+                `SELECT ${join.columns} FROM (SELECT ? AS \`sub_id\`) x ${join.join}`,
+                [subId],
+            );
+
+        const [found] = await read(row.subId);
+        expect(found).toMatchObject({
+            sub_suspended_at: null,
+            sub_suspended_reason: null,
+            sub_handler_name: 'tick',
+        });
+        expect(Number(found.sub_row_id)).toBeGreaterThan(0);
+        expect(found.sub_context).toBe('{"region":"eu"}');
+
+        const [missing] = await read('no-such-sub');
+        expect(missing).toMatchObject({
+            sub_row_id: null,
+            sub_handler_name: null,
+        });
+    });
+
+    it('tell a schedule whose row is gone', async () => {
+        const { uid, row } = await createScheduleRow();
+        const absent = async (subId: string) => {
+            const [result] = await env.server.clients.db.pread(
+                `SELECT CASE WHEN ${durable().scheduleSubscriptionAbsent('?')} THEN 1 ELSE 0 END AS \`absent\``,
+                [subId],
+            );
+            return Number(result.absent);
+        };
+
+        await expect(absent(row.subId)).resolves.toBe(0);
+        await durable().removeScheduleSubscription(uid);
+        await expect(absent(row.subId)).resolves.toBe(1);
+    });
+});

src/backend/stores/events/DurableSubscriptionStore.ts

@@ -23,7 +23,9 @@ import {
     EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
     type TieredQuota,
 } from '../../controllers/events/limits.js';
+import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
 import { HttpError } from '../../core/http/HttpError.js';
+import type { SqlGuard } from '../../core/schedules.js';
 import type { DeliveryClass } from '../../services/events/registry.js';
 import type { FsOp } from '../../services/events/subjects.js';
 import {
@@ -37,7 +39,13 @@ import {
     EVENT_SUBSCRIPTION_WIDTHS,
 } from './columnWidths.js';
 import type { GenerationBump } from './EventSubscriptionStore.js';
-import type { DurableSubscription, SubscriptionPermission } from './types.js';
+import {
+    isScheduleRow,
+    SCHEDULE_TOKEN_PREFIX,
+    scheduleToken,
+    type DurableSubscription,
+    type SubscriptionPermission,
+} from './types.js';
 
 /**
  * Subscriptions that outlive the connection that made them.
@@ -66,6 +74,9 @@ export const DURABLE_CONTEXT_MAX_BYTES = 4096;
 export const DURABLE_LIST_DEFAULT_LIMIT = 50;
 export const DURABLE_LIST_LIMIT_CAP = 200;
 
+/** Rows one page of a holder-wide settle takes. */
+export const HOLDER_SETTLE_BATCH = 500;
+
 /**
  * Rows one handler-lifecycle pass takes. A removal suspends its dependents in
  * batches so a widely-used name cannot make one call hold the whole set.
@@ -101,6 +112,15 @@ export interface DurableSubscriptionInput {
     limits?: TieredQuota;
 }
 
+/**
+ * What a reap deleted, and the generation it bumped per owner. The caller
+ * publishes the bumps, since the reap only invalidates this region.
+ */
+export interface DurableReap {
+    rows: DurableSubscription[];
+    bumps: GenerationBump[];
+}
+
 /** One keyset page of rows, and where the next page starts. */
 export interface DurablePage {
     rows: DurableSubscription[];
@@ -148,6 +168,13 @@ const TERMINAL_CLAUSE =
     '(`suspended_at` IS NOT NULL AND `suspended_reason` IN (' +
     `${TERMINAL_SUSPENDED_REASONS.map(() => '?').join(', ')}))`;
 
+/**
+ * Leaves out the rows schedules own: they have their own caps and listing, and
+ * never enter the dispatch cache. Binds {@link SCHEDULE_TOKEN_PATTERN}.
+ */
+const NOT_SCHEDULE_CLAUSE = '`token` NOT LIKE ?';
+const SCHEDULE_TOKEN_PATTERN = `${SCHEDULE_TOKEN_PREFIX}%`;
+
 /** Where a row is moving to when its anchor is deleted under it. */
 export interface ReanchorInput {
     token: string;
@@ -167,6 +194,12 @@ const contextTooLarge = (): HttpError =>
         { legacyCode: 'events_context_too_large' },
     );
 
+export const assertDurableContext = (context: string | null): void => {
+    if (context === null) return;
+    if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
+        throw contextTooLarge();
+};
+
 const quotaReached = (limit: number, scope: 'account' | 'app'): HttpError =>
     new HttpError(
         429,
@@ -207,6 +240,16 @@ const quotaRefusal = (
     return null;
 };
 
+/**
+ * A created row; `reaped` holds the bumps of terminal rows it trimmed. A
+ * schedule's row is never cached, so it comes back with no bump.
+ */
+export interface DurableCreated {
+    row: DurableSubscription;
+    bump: GenerationBump | null;
+    reaped: GenerationBump[];
+}
+
 /**
  * The quota refusal for a create that concurrent creates beat to the last slot
  * after its own insert. Its row is already gone again; `bump` still has to be
@@ -291,11 +334,13 @@ export class DurableSubscriptionStore extends PuterStore {
      * Insert one subscription and put it in this region's cache before
      * returning. The generation bump is what tells every other region, and the
      * caller broadcasts it.
+     *
+     * A schedule's row skips the durable quotas and the cache, so it comes back
+     * with no bump: its caps are the schedule store's.
      */
-    async create(
-        input: DurableSubscriptionInput,
-    ): Promise<{ row: DurableSubscription; bump: GenerationBump }> {
-        this.#assertContext(input.context);
+    async create(input: DurableSubscriptionInput): Promise<DurableCreated> {
+        assertDurableContext(input.context);
+        const forSchedule = isScheduleRow(input);
         assertColumnWidths(EVENT_SUBSCRIPTION_WIDTHS, {
             token: input.token,
             appUid: input.appUid,
@@ -307,12 +352,14 @@ export class DurableSubscriptionStore extends PuterStore {
             permission: input.permission,
         });
 
-        const refusal = quotaRefusal(
-            input,
-            await this.countForHolder(input.holderUserId, input.appUid),
-            0,
-        );
-        if (refusal) throw refusal;
+        if (!forSchedule) {
+            const refusal = quotaRefusal(
+                input,
+                await this.countForHolder(input.holderUserId, input.appUid),
+                0,
+            );
+            if (refusal) throw refusal;
+        }
 
         const row: DurableSubscription = {
             durable: true,
@@ -358,6 +405,7 @@ export class DurableSubscriptionStore extends PuterStore {
             expires_at: row.expiresAt,
             created_at: row.createdAt,
         });
+        if (forSchedule) return { row, bump: null, reaped: [] };
 
         // Concurrent creates can all pass the count above. Counted again with
         // this row in it, each one past the cap takes itself back out — at
@@ -368,18 +416,20 @@ export class DurableSubscriptionStore extends PuterStore {
         );
         const lost = quotaRefusal(input, held, 1);
         if (lost) throw new DurableQuotaRaceLost(lost, await this.remove(row));
-        if (held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT)
-            await this.#dropOldestTerminal(
-                input.holderUserId,
-                held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
-            );
+        const reaped =
+            held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT
+                ? await this.#dropOldestTerminal(
+                      input.holderUserId,
+                      held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
+                  )
+                : [];
 
         // Write-through, and over the whole owner rather than the one row: it
         // costs the same indexed read as a warm would, on a path rate-limited
         // to a few calls a minute, and it leaves this region needing nothing
         // from the table before it can deliver.
         await this.#rebuildRegion(row.ownerUserId);
-        return { row, bump: await this.#bump(row.ownerUserId) };
+        return { row, bump: await this.#bump(row.ownerUserId), reaped };
     }
 
     /**
@@ -439,8 +489,9 @@ export class DurableSubscriptionStore extends PuterStore {
                 });
         }
 
-        await this.stores.eventSubscription.dropDurables(suspended);
-        const owners = new Set(suspended.map((row) => row.ownerUserId));
+        const cached = suspended.filter((row) => !isScheduleRow(row));
+        await this.stores.eventSubscription.dropDurables(cached);
+        const owners = new Set(cached.map((row) => row.ownerUserId));
         const bumps = await Promise.all(
             [...owners].map((owner) => this.#bump(owner)),
         );
@@ -468,6 +519,7 @@ export class DurableSubscriptionStore extends PuterStore {
         // owner from the first row it is given.
         const byOwner = new Map<number, DurableSubscription[]>();
         for (const row of rows) {
+            if (isScheduleRow(row)) continue;
             const live: DurableSubscription = {
                 ...row,
                 suspendedAt: null,
@@ -541,7 +593,7 @@ export class DurableSubscriptionStore extends PuterStore {
         row: DurableSubscription,
         context: string | null,
     ): Promise<{ row: DurableSubscription; bump: GenerationBump } | null> {
-        this.#assertContext(context);
+        assertDurableContext(context);
         const written = await this.clients.db.write(
             `UPDATE \`${TABLE}\` SET \`context\` = ? WHERE \`sub_id\` = ?`,
             [context, row.subId],
@@ -573,8 +625,12 @@ export class DurableSubscriptionStore extends PuterStore {
      * Reap rows past their expiry, dropping each from the cache so the region
      * stops delivering against it without waiting for a rebuild.
      */
-    async sweepExpired(batchSize: number): Promise<number> {
-        return this.#reap(await this.#listExpired(nowSeconds(), batchSize));
+    async sweepExpired(batchSize: number): Promise<DurableReap> {
+        const cutoff = nowSeconds();
+        return this.#reap(await this.#listExpired(cutoff, batchSize), {
+            where: '`expires_at` IS NOT NULL AND `expires_at` <= ?',
+            values: [cutoff],
+        });
     }
 
     /**
@@ -582,8 +638,14 @@ export class DurableSubscriptionStore extends PuterStore {
      * never resumes is a row kept only so its holder can see why it stopped,
      * and that answer has a shelf life.
      */
-    async sweepSuspended(cutoff: number, batchSize: number): Promise<number> {
-        return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize));
+    async sweepSuspended(
+        cutoff: number,
+        batchSize: number,
+    ): Promise<DurableReap> {
+        return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize), {
+            where: '`suspended_at` IS NOT NULL AND `suspended_at` <= ?',
+            values: [cutoff],
+        });
     }
 
     // -- Reads -------------------------------------------------------
@@ -622,8 +684,16 @@ export class DurableSubscriptionStore extends PuterStore {
 
         // The scope half of the predicate is what the total counts over; the
         // cursor half only positions one page inside it.
-        const scope = ['`holder_user_id` = ?', this.#unexpiredClause()];
-        const scopeParams: unknown[] = [holderUserId, nowSeconds()];
+        const scope = [
+            '`holder_user_id` = ?',
+            this.#unexpiredClause(),
+            NOT_SCHEDULE_CLAUSE,
+        ];
+        const scopeParams: unknown[] = [
+            holderUserId,
+            nowSeconds(),
+            SCHEDULE_TOKEN_PATTERN,
+        ];
         if (options.appUid !== undefined && options.appUid !== null) {
             scope.push('`app_uid` = ?');
             scopeParams.push(options.appUid);
@@ -675,7 +745,8 @@ export class DurableSubscriptionStore extends PuterStore {
                 `SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 0 ` +
                 'WHEN `app_uid` = ? THEN 1 ELSE 0 END) AS `for_app`, ' +
                 `SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 1 ELSE 0 END) AS \`terminal\` ` +
-                `FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()}`,
+                `FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()} ` +
+                `AND ${NOT_SCHEDULE_CLAUSE}`,
             [
                 ...terminal,
                 ...terminal,
@@ -683,6 +754,7 @@ export class DurableSubscriptionStore extends PuterStore {
                 ...terminal,
                 holderUserId,
                 nowSeconds(),
+                SCHEDULE_TOKEN_PATTERN,
             ],
         );
         return {
@@ -693,22 +765,26 @@ export class DurableSubscriptionStore extends PuterStore {
     }
 
     /**
-     * The holder's live rows, which is what a revocation has to consider. Same
-     * index the listing and the quota use — passing `appUid` narrows to one
-     * app's rows, which is what a grant made to that app can have authorized.
+     * The holder's live rows, which is what a revocation has to consider, one
+     * keyset page at a time. Same index the listing and the quota use — passing
+     * `appUid` narrows to one app's rows, which is what a grant made to that
+     * app can have authorized. Schedule rows are included: a holder can hold
+     * far more of them than the durable quota allows, which is why this pages.
      *
      * `includeSuspended` also returns rows out of service for a reason that
      * lifts. A revocation needs them: those reasons hold a backlog, and a row
      * skipped here comes back in service on the next resume and hands it over.
+     * Rows suspended for good are left out.
      *
-     * Bounded by the per-account quota, so the whole set fits one read. Rows
-     * suspended for good hold no slot, so they are left out.
+     * `excludeSchedules` is for a settle no schedule row can match.
      */
-    async listActiveForHolder(
+    async listActiveForHolderPage(
         holderUserId: number,
         appUid: string | null,
-        opts: { includeSuspended?: boolean } = {},
-    ): Promise<DurableSubscription[]> {
+        afterId: number,
+        batchSize: number = HOLDER_SETTLE_BATCH,
+        opts: { includeSuspended?: boolean; excludeSchedules?: boolean } = {},
+    ): Promise<DurablePage> {
         const where = [
             '`holder_user_id` = ?',
             opts.includeSuspended
@@ -721,16 +797,17 @@ export class DurableSubscriptionStore extends PuterStore {
             ...(opts.includeSuspended ? TERMINAL_SUSPENDED_REASONS : []),
             nowSeconds(),
         ];
+        if (opts.excludeSchedules) {
+            where.push(NOT_SCHEDULE_CLAUSE);
+            params.push(SCHEDULE_TOKEN_PATTERN);
+        }
         if (appUid !== null) {
             where.push('`app_uid` = ?');
             params.push(appUid);
         }
-        const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                `WHERE ${where.join(' AND ')} ORDER BY \`id\` LIMIT ?`,
-            [...params, EVENTS_DURABLE_SUBSCRIPTIONS_MAX],
-        );
-        return rows.map(toRow);
+        return this.#page(where, params, afterId, batchSize, {
+            primary: true,
+        });
     }
 
     /**
@@ -768,25 +845,27 @@ export class DurableSubscriptionStore extends PuterStore {
 
     /**
      * One holder's rows suspended for a given reason — what a resume condition
-     * that belongs to the account rather than to a handler releases.
+     * that belongs to the account rather than to a handler releases — one
+     * keyset page at a time.
      */
-    async listSuspendedForHolder(
+    async listSuspendedForHolderPage(
         holderUserId: number,
         reason: SuspendedReason,
-    ): Promise<DurableSubscription[]> {
-        const rows = await this.clients.db.pread(
-            `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
-                'WHERE `holder_user_id` = ? AND `suspended_at` IS NOT NULL ' +
-                `AND \`suspended_reason\` = ? AND ${this.#unexpiredClause()} ` +
-                'ORDER BY `id` LIMIT ?',
+        afterId: number,
+        batchSize: number = HOLDER_SETTLE_BATCH,
+    ): Promise<DurablePage> {
+        return this.#page(
             [
-                holderUserId,
-                reason,
-                nowSeconds(),
-                EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
+                '`holder_user_id` = ?',
+                '`suspended_at` IS NOT NULL',
+                '`suspended_reason` = ?',
+                this.#unexpiredClause(),
             ],
+            [holderUserId, reason, nowSeconds()],
+            afterId,
+            batchSize,
+            { primary: true },
         );
-        return rows.map(toRow);
     }
 
     /**
@@ -819,8 +898,8 @@ export class DurableSubscriptionStore extends PuterStore {
         const rows = await this.clients.db.pread(
             `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                 'WHERE `owner_user_id` = ? AND `suspended_at` IS NULL AND ' +
-                `${this.#unexpiredClause()}`,
-            [ownerUserId, nowSeconds()],
+                `${this.#unexpiredClause()} AND ${NOT_SCHEDULE_CLAUSE}`,
+            [ownerUserId, nowSeconds(), SCHEDULE_TOKEN_PATTERN],
         );
         return rows.map(toRow);
     }
@@ -846,30 +925,96 @@ export class DurableSubscriptionStore extends PuterStore {
      * What app deletion reaps: `#reap` already purges the backlog, drops the
      * cache and bumps a generation per owner, so this is the whole of it.
      */
-    async reapForApp(appUid: string, batchSize: number): Promise<number> {
+    async reapForApp(appUid: string, batchSize: number): Promise<DurableReap> {
         return this.#reap(await this.#listForApp(appUid, batchSize));
     }
 
+    /**
+     * Delete the row a schedule owns, whether or not the schedule ever recorded
+     * its id: a create that crashed between the two inserts leaves one it
+     * didn't. Never cached, so there is nothing to drop or bump. Called by
+     * extensions.
+     */
+    async removeScheduleSubscription(scheduleUid: string): Promise<boolean> {
+        const written = await this.clients.db.write(
+            `DELETE FROM \`${TABLE}\` WHERE \`token\` = ?`,
+            [scheduleToken(scheduleUid)],
+        );
+        return written.anyRowsAffected;
+    }
+
+    // Statements and fragments for the schedules' own transactions and reads,
+    // so no SQL outside this store names the table. Called by extensions.
+
+    /**
+     * Delete the row a schedule owns, only while `guard` holds: for a batch
+     * that ends the schedule in the same transaction.
+     */
+    scheduleSubscriptionDeleteEntry(
+        scheduleUid: string,
+        guard: SqlGuard,
+    ): BatchEntry {
+        return {
+            statement:
+                `DELETE FROM \`${TABLE}\` WHERE \`token\` = ? ` +
+                `AND EXISTS (SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where})`,
+            values: [scheduleToken(scheduleUid), ...guard.values],
+        };
+    }
+
+    /**
+     * A `LEFT JOIN` of the row a schedule owns, as `alias`, on `subIdColumn`,
+     * and the columns it selects: `sub_row_id`, `sub_suspended_at`,
+     * `sub_suspended_reason`, `sub_handler_name` and `sub_context`.
+     */
+    scheduleSubscriptionJoin(
+        alias: string,
+        subIdColumn: string,
+    ): { columns: string; join: string } {
+        return {
+            columns:
+                `${alias}.\`id\` AS \`sub_row_id\`, ` +
+                `${alias}.\`suspended_at\` AS \`sub_suspended_at\`, ` +
+                `${alias}.\`suspended_reason\` AS \`sub_suspended_reason\`, ` +
+                `${alias}.\`handler_name\` AS \`sub_handler_name\`, ` +
+                `${alias}.\`context\` AS \`sub_context\``,
+            join: `LEFT JOIN \`${TABLE}\` ${alias} ON ${alias}.\`sub_id\` = ${subIdColumn}`,
+        };
+    }
+
+    /** A condition true when no subscription row has the id `subIdExpr`. */
+    scheduleSubscriptionAbsent(subIdExpr: string): string {
+        return `NOT EXISTS (SELECT 1 FROM \`${TABLE}\` e WHERE e.\`sub_id\` = ${subIdExpr})`;
+    }
+
     // -- Internals ---------------------------------------------------
 
     /**
-     * Delete a holder's oldest rows suspended for good, past what `list` keeps.
-     * Best-effort: the next subscribe tries again.
+     * Delete a holder's oldest rows suspended for good, past what `list` keeps,
+     * and return the reap's bumps. Best-effort: the next subscribe tries
+     * again.
      */
     async #dropOldestTerminal(
         holderUserId: number,
         count: number,
-    ): Promise<void> {
+    ): Promise<GenerationBump[]> {
         try {
             const rows = await this.clients.db.pread(
                 `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                     `WHERE \`holder_user_id\` = ? AND ${TERMINAL_CLAUSE} ` +
+                    `AND ${NOT_SCHEDULE_CLAUSE} ` +
                     'ORDER BY `suspended_at`, `id` LIMIT ?',
-                [holderUserId, ...TERMINAL_SUSPENDED_REASONS, count],
+                [
+                    holderUserId,
+                    ...TERMINAL_SUSPENDED_REASONS,
+                    SCHEDULE_TOKEN_PATTERN,
+                    count,
+                ],
             );
-            await this.#reap(rows.map(toRow));
+            return (await this.#reap(rows.map(toRow))).bumps;
         } catch (err) {
             console.warn('[events] could not drop old revoked rows', err);
+            return [];
         }
     }
 
@@ -891,15 +1036,23 @@ export class DurableSubscriptionStore extends PuterStore {
         return rows.map(toRow);
     }
 
-    /** One page of a whole-table scan, ordered and positioned by primary key. */
+    /**
+     * One keyset page, ordered and positioned by primary key. The replica
+     * unless `primary`, which settles need: a lagging read would skip a row
+     * just created.
+     */
     async #page(
         where: string[],
         params: unknown[],
         afterId: number,
         batchSize: number,
+        opts: { primary?: boolean } = {},
     ): Promise<DurablePage> {
         const limit = Math.max(1, Math.floor(batchSize));
-        const rows = await this.clients.db.read(
+        const read = opts.primary
+            ? this.clients.db.pread.bind(this.clients.db)
+            : this.clients.db.read.bind(this.clients.db);
+        const rows = await read(
             `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
                 `WHERE ${[...where, '`id` > ?'].join(' AND ')} ` +
                 'ORDER BY `id` LIMIT ?',
@@ -952,30 +1105,56 @@ export class DurableSubscriptionStore extends PuterStore {
         return rows.map(toRow);
     }
 
-    /** Delete a batch and leave no region delivering against any of it. */
-    async #reap(rows: readonly DurableSubscription[]): Promise<number> {
-        if (rows.length === 0) return 0;
+    /**
+     * Delete a batch and stop this region delivering against any of it. Other
+     * regions stop once the caller publishes the returned bumps; the deleted
+     * rows come back so the caller can end the schedules among them.
+     *
+     * `stillDue` re-checks candidates read from a replica on the primary, so a
+     * row resumed or extended since is kept and not reported.
+     */
+    async #reap(
+        candidates: readonly DurableSubscription[],
+        stillDue?: { where: string; values: unknown[] },
+    ): Promise<DurableReap> {
+        if (candidates.length === 0) return { rows: [], bumps: [] };
 
+        const ids = candidates.map((row) => row.subId);
+        const inList = `(${ids.map(() => '?').join(', ')})`;
         await this.clients.db.write(
-            `DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ` +
-                `(${rows.map(() => '?').join(', ')})`,
-            rows.map((row) => row.subId),
+            `DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}` +
+                (stillDue ? ` AND ${stillDue.where}` : ''),
+            [...ids, ...(stillDue?.values ?? [])],
         );
-
-        await this.stores.eventSubscription.dropDurables(rows);
-        for (const row of rows) {
-            // Whatever was still owed to a row that no longer exists.
-            await this.stores.pendingDelivery.purge(row.subId).catch(() => {});
+        let rows = candidates;
+        if (stillDue) {
+            const kept = new Set(
+                (
+                    await this.clients.db.pread(
+                        `SELECT \`sub_id\` FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}`,
+                        ids,
+                    )
+                ).map((row) => String(row.sub_id)),
+            );
+            rows = candidates.filter((row) => !kept.has(row.subId));
+            if (rows.length === 0) return { rows: [], bumps: [] };
         }
-        const owners = new Set(rows.map((row) => row.ownerUserId));
-        for (const ownerUserId of owners) await this.#bump(ownerUserId);
-        return rows.length;
-    }
 
-    #assertContext(context: string | null): void {
-        if (context === null) return;
-        if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
-            throw contextTooLarge();
+        const cached = rows.filter((row) => !isScheduleRow(row));
+        await this.stores.eventSubscription.dropDurables(cached);
+        for (const row of cached) {
+            // Whatever was still owed to a row that no longer exists.
+            try {
+                await this.stores.pendingDelivery.purge(row.subId);
+            } catch {
+                // The pending sweep purges it once it finds the row gone.
+            }
+        }
+        const owners = new Set(cached.map((row) => row.ownerUserId));
+        const bumps: GenerationBump[] = [];
+        for (const ownerUserId of owners)
+            bumps.push(await this.#bump(ownerUserId));
+        return { rows: [...rows], bumps };
     }
 
     async #bump(ownerUserId: number): Promise<GenerationBump> {

src/backend/stores/events/EventHandlerStore.integration.test.ts

@@ -55,7 +55,10 @@ const OTHER_SOURCE = 'async ({ event, ctx }) => { console.log(ctx.url); }';
 const codeOf = (code: string) => (err: unknown) =>
     isHttpError(err) && err.legacyCode === code;
 
-/** A minimal `apps` row, owned by `ownerUserId`, for the events-workers reads to join against. */
+/**
+ * A minimal `apps` row, owned by `ownerUserId`, for the events-workers reads to
+ * join against.
+ */
 const makeApp = async (owner: number): Promise<string> => {
     const uid = `app-${uuidv4()}`;
     await env.server.clients.db.write(
@@ -122,7 +125,11 @@ describe('publishing a handler', () => {
     });
 
     it('refuses different source from a publisher that did not name the base', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
 
         await expect(
             handlers().publish({
@@ -179,7 +186,11 @@ describe('publishing a handler', () => {
     });
 
     it('takes the name outright for a publisher that asked to replace', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
 
         const replaced = await handlers().publish({
             appUid,
@@ -236,9 +247,10 @@ describe('publishing a handler', () => {
         // The row reflects whichever one actually won — never a mix of the
         // two, and never silently both.
         const stored = await handlers().getByName(appUid, 'ingestUpload');
-        expect([OTHER_SOURCE, 'async () => { /* the other racer */ }']).toContain(
-            stored!.source,
-        );
+        expect([
+            OTHER_SOURCE,
+            'async () => { /* the other racer */ }',
+        ]).toContain(stored!.source);
     });
 
     it('never lets a create-vs-create race leak a raw database error', async () => {
@@ -271,7 +283,11 @@ describe('publishing a handler', () => {
     });
 
     it('scopes the name to one app', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
         const other = await handlers().publish({
             appUid: otherAppUid,
             name: 'ingestUpload',
@@ -285,7 +301,13 @@ describe('publishing a handler', () => {
     });
 
     it('refuses a name that is not an addressable identifier', async () => {
-        for (const name of ['', ' leading', 'has space', '-dash', 'x'.repeat(129)]) {
+        for (const name of [
+            '',
+            ' leading',
+            'has space',
+            '-dash',
+            'x'.repeat(129),
+        ]) {
             await expect(
                 handlers().publish({ appUid, name, source: SOURCE }),
             ).rejects.toSatisfy(codeOf('events_handler_name_invalid'));
@@ -309,7 +331,11 @@ describe('publishing a handler', () => {
 
 describe('listing handlers', () => {
     it('reports names, hashes and dependents, never source', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
         await handlers().publish({
             appUid,
             name: 'indexDocument',
@@ -327,13 +353,15 @@ describe('listing handlers', () => {
             hash: hashContent(SOURCE),
             subscriptions: 0,
         });
-        expect(
-            listed.every((row) => !('source' in row)),
-        ).toBe(true);
+        expect(listed.every((row) => !('source' in row))).toBe(true);
     });
 
     it('counts the subscriptions each name is carrying', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
         for (const suffix of ['a', 'b']) {
             await env.server.clients.db.write(
                 'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, ' +
@@ -365,7 +393,11 @@ describe('listing handlers', () => {
 
 describe('removing a handler', () => {
     it('drops the row and reports what it was', async () => {
-        await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE });
+        await handlers().publish({
+            appUid,
+            name: 'ingestUpload',
+            source: SOURCE,
+        });
 
         await expect(
             handlers().remove(appUid, 'ingestUpload'),
@@ -437,6 +469,136 @@ describe('the set a delivery addresses', () => {
         }
     });
 
+    it('reads a miss from the replica when asked to, and caches it', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        const pread = vi.spyOn(env.server.clients.db, 'pread');
+        const read = vi.spyOn(env.server.clients.db, 'read');
+        try {
+            const first = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(first.map((h) => h.name)).toEqual(['a']);
+            await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(pread).not.toHaveBeenCalled();
+            expect(read).toHaveBeenCalledTimes(1);
+        } finally {
+            pread.mockRestore();
+            read.mockRestore();
+        }
+    });
+
+    it('never hands the delivery path a set a lagging replica read cached', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        const fresh = 'async ({ event }) => event.path';
+        // The replica still has the old source when a schedule run reads it,
+        // right after a publish dropped the caches.
+        const stale = await handlers().setForApp(cachedApp);
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: fresh,
+            replace: true,
+        });
+        const read = vi
+            .spyOn(env.server.clients.db, 'read')
+            .mockResolvedValueOnce(
+                stale.map((h) => ({ name: h.name, source_hash: h.sourceHash })),
+            );
+        try {
+            const viaReplica = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(viaReplica[0].sourceHash).toBe(hashContent(SOURCE));
+        } finally {
+            read.mockRestore();
+        }
+        const delivery = await handlers().cachedSetForApp(cachedApp);
+        expect(delivery[0].sourceHash).toBe(hashContent(fresh));
+
+        // A publish drops the replica-read entry too.
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+            replace: true,
+        });
+        const again = await handlers().cachedSetForApp(cachedApp, {
+            missRead: 'replica',
+        });
+        expect(again[0].sourceHash).toBe(hashContent(SOURCE));
+    });
+
+    it('checks an empty replica read on the primary', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        // The replica hasn't seen the publish yet.
+        const read = vi
+            .spyOn(env.server.clients.db, 'read')
+            .mockResolvedValueOnce([]);
+        const pread = vi.spyOn(env.server.clients.db, 'pread');
+        try {
+            const set = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(set.map((h) => h.name)).toEqual(['a']);
+            expect(pread).toHaveBeenCalledTimes(1);
+
+            // The primary's answer is cached for the next run. (SQLite's
+            // `pread` goes through `read`, so count from here.)
+            const reads = read.mock.calls.length;
+            await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+            });
+            expect(read).toHaveBeenCalledTimes(reads);
+            expect(pread).toHaveBeenCalledTimes(1);
+        } finally {
+            read.mockRestore();
+            pread.mockRestore();
+        }
+    });
+
+    it('re-reads on the primary a replica set without the handler a run needs', async () => {
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'a',
+            source: SOURCE,
+        });
+        const stale = await handlers().setForApp(cachedApp);
+        await handlers().publish({
+            appUid: cachedApp,
+            name: 'b',
+            source: SOURCE,
+        });
+        // The replica has only `a`, published before `b`.
+        const read = vi
+            .spyOn(env.server.clients.db, 'read')
+            .mockResolvedValueOnce(
+                stale.map((h) => ({ name: h.name, source_hash: h.sourceHash })),
+            );
+        try {
+            const set = await handlers().cachedSetForApp(cachedApp, {
+                missRead: 'replica',
+                handlerName: 'b',
+            });
+            expect(set.map((h) => h.name).sort()).toEqual(['a', 'b']);
+        } finally {
+            read.mockRestore();
+        }
+    });
+
     it('follows every publish and removal', async () => {
         await handlers().publish({
             appUid: cachedApp,
@@ -495,7 +657,12 @@ describe('the set a delivery addresses', () => {
 
             expect(emit).toHaveBeenCalledWith(
                 'outer.cacheUpdate',
-                { cacheKey: [`eventHandlers:set:${cachedApp}`] },
+                {
+                    cacheKey: [
+                        `eventHandlers:set:${cachedApp}`,
+                        `eventHandlers:set:${cachedApp}:r`,
+                    ],
+                },
                 {},
             );
         } finally {
@@ -627,7 +794,11 @@ describe('listing an owner`s events workers', () => {
     it('reports one row per app, with its handler count and app details', async () => {
         const uid = await makeApp(ownerUserId);
         await handlers().publish({ appUid: uid, name: 'a', source: SOURCE });
-        await handlers().publish({ appUid: uid, name: 'b', source: OTHER_SOURCE });
+        await handlers().publish({
+            appUid: uid,
+            name: 'b',
+            source: OTHER_SOURCE,
+        });
 
         const page = await handlers().listEventsWorkersForOwner(ownerUserId);
 
@@ -654,7 +825,11 @@ describe('listing an owner`s events workers', () => {
         const uidA = await makeApp(ownerUserId);
         const uidB = await makeApp(ownerUserId);
         await handlers().publish({ appUid: uidA, name: 'a', source: SOURCE });
-        await handlers().publish({ appUid: uidB, name: 'b', source: OTHER_SOURCE });
+        await handlers().publish({
+            appUid: uidB,
+            name: 'b',
+            source: OTHER_SOURCE,
+        });
 
         const page = await handlers().listEventsWorkersForOwner(ownerUserId, {
             appUid: uidA,
@@ -678,7 +853,11 @@ describe('listing an owner`s events workers', () => {
         const uids: string[] = [];
         for (let i = 0; i < 3; i++) {
             const uid = await makeApp(ownerUserId);
-            await handlers().publish({ appUid: uid, name: 'a', source: SOURCE });
+            await handlers().publish({
+                appUid: uid,
+                name: 'a',
+                source: SOURCE,
+            });
             uids.push(uid);
         }
 
@@ -696,9 +875,7 @@ describe('listing an owner`s events workers', () => {
         expect(second.cursor).toBeUndefined();
 
         expect(
-            [...first.items, ...second.items]
-                .map((row) => row.appUid)
-                .sort(),
+            [...first.items, ...second.items].map((row) => row.appUid).sort(),
         ).toEqual([...uids].sort());
     });
 });

src/backend/stores/events/EventHandlerStore.ts

@@ -73,6 +73,13 @@ export const HANDLER_SET_CACHE_TTL_SECONDS = 30;
 const handlerSetCacheKey = (appUid: string): string =>
     `eventHandlers:set:${appUid}`;
 
+/**
+ * The same set as read from the replica, kept apart so a lagging read can never
+ * stand in for the primary's answer on the delivery path.
+ */
+const replicaSetCacheKey = (appUid: string): string =>
+    `${handlerSetCacheKey(appUid)}:r`;
+
 // -- Shapes -----------------------------------------------------------
 
 /** A row in `event_handlers`, as the rest of the system sees it. */
@@ -337,13 +344,16 @@ export class EventHandlerStore extends PuterStore {
      */
     async setForApp(
         appUid: string,
+        options: { replica?: boolean } = {},
     ): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
-        const rows = await this.clients.db.pread(
+        const sql =
             'SELECT `name`, `source_hash` ' +
-                `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
-                'ORDER BY `name` LIMIT ?',
-            [appUid, EVENTS_HANDLERS_PER_APP],
-        );
+            `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
+            'ORDER BY `name` LIMIT ?';
+        const params = [appUid, EVENTS_HANDLERS_PER_APP];
+        const rows = options.replica
+            ? await this.clients.db.read(sql, params)
+            : await this.clients.db.pread(sql, params);
         return rows.map((row) => ({
             name: String(row.name),
             sourceHash: String(row.source_hash),
@@ -354,23 +364,43 @@ export class EventHandlerStore extends PuterStore {
      * `setForApp` for the delivery path, which asks once per invocation. Cached
      * for up to {@link HANDLER_SET_CACHE_TTL_SECONDS}, and dropped everywhere by
      * every write here. An empty set is not cached: it fails the delivery, and
-     * must not outlast the publish that fills it.
+     * must not outlast the publish that fills it. Schedule runs read a miss
+     * from the replica (`missRead: 'replica'`), since they come by the clock
+     * rather than after a write, and cache it under a key of their own; an
+     * empty replica read is checked on the primary, since a lagging replica
+     * right after a publish would fail the run. Called by extensions.
      */
     async cachedSetForApp(
         appUid: string,
+        options: {
+            missRead?: 'primary' | 'replica';
+            handlerName?: string;
+        } = {},
     ): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
-        const key = handlerSetCacheKey(appUid);
+        const replica = options.missRead === 'replica';
+        // A set without the handler may be from before its publish.
+        const usable = (set: Pick<EventHandler, 'name'>[]) =>
+            set.length > 0 &&
+            (options.handlerName === undefined ||
+                set.some((handler) => handler.name === options.handlerName));
+        const key = replica
+            ? replicaSetCacheKey(appUid)
+            : handlerSetCacheKey(appUid);
         try {
             const cached = JSON.parse(
                 (await this.clients.redis.get(key)) ?? 'null',
             ) as unknown;
-            if (Array.isArray(cached) && cached.length > 0)
+            if (
+                Array.isArray(cached) &&
+                usable(cached as Pick<EventHandler, 'name'>[])
+            )
                 return cached as Pick<EventHandler, 'name' | 'sourceHash'>[];
         } catch {
             // Read the table instead.
         }
 
-        const set = await this.setForApp(appUid);
+        let set = await this.setForApp(appUid, { replica });
+        if (replica && !usable(set)) set = await this.setForApp(appUid);
         if (set.length > 0) {
             try {
                 await this.clients.redis.set(
@@ -616,7 +646,7 @@ export class EventHandlerStore extends PuterStore {
     /** Drop the delivery path's cached set here and on every peer. */
     async #forgetSet(appUid: string): Promise<void> {
         await this.publishCacheKeys({
-            keys: [handlerSetCacheKey(appUid)],
+            keys: [handlerSetCacheKey(appUid), replicaSetCacheKey(appUid)],
             broadcast: true,
         });
     }

src/backend/stores/events/KvShareHandleStore.integration.test.ts

@@ -326,7 +326,7 @@ describe('the notify flag', () => {
 });
 
 describe('live handles by owner and app', () => {
-    it('lists only the owner\'s live handles in that app, oldest first', async () => {
+    it("lists only the owner's live handles in that app, oldest first", async () => {
         const appUid = 'app-owner-listing';
         const first = await store().mint(
             input({ appUid, keyPrefix: 'owned-a:' }),
@@ -502,13 +502,11 @@ describe('the live cache across nodes', () => {
         expect(await store().getLive(row.handle)).toBeNull();
     });
 
-    it('drops an owner\'s cached rows when the owner is suspended, here and on other nodes', async () => {
+    it("drops an owner's cached rows when the owner is suspended, here and on other nodes", async () => {
         const row = await store().mint(
             input({ appUid: 'app-forget', keyPrefix: 'forget-owner:' }),
         );
-        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
-            false,
-        );
+        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
         const setSuspended = (value: number) =>
             env.server.clients.db.write(
                 'UPDATE `user` SET `suspended` = ? WHERE `id` = ?',
@@ -534,11 +532,7 @@ describe('the live cache across nodes', () => {
                     ([key]) => key === 'outer.pubsub.kv.share.forgetOwner',
                 ),
             ).toEqual([
-                [
-                    'outer.pubsub.kv.share.forgetOwner',
-                    { ownerUserId },
-                    {},
-                ],
+                ['outer.pubsub.kv.share.forgetOwner', { ownerUserId }, {}],
             ]);
 
             // Another node lifting the suspension reaches this one's cache.
@@ -557,13 +551,11 @@ describe('the live cache across nodes', () => {
         }
     });
 
-    it('drops an owner\'s cached rows when the user store suspends or unsuspends them', async () => {
+    it("drops an owner's cached rows when the user store suspends or unsuspends them", async () => {
         const row = await store().mint(
             input({ appUid: 'app-forget', keyPrefix: 'forget-store:' }),
         );
-        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
-            false,
-        );
+        expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
         try {
             await env.server.stores.user.update(ownerUserId, {
                 suspended: true,
@@ -895,21 +887,27 @@ describe('retired handle retention', () => {
         expect(await exists(live)).toBe(true);
     });
 
-    it('purges once per day across nodes, keeping live and recent rows', async () => {
+    it('purges retired handles past the retention, keeping live and recent rows', async () => {
         const kvShare = env.server.services.kvShare;
-        await env.server.clients.redis.del('kvShare:purge:gate');
         const old = await retiredAt('purge-old:', now() - 31 * DAY);
         const recent = await retiredAt('purge-recent:', now() - 29 * DAY);
         const live = await retiredAt('purge-live:', null);
 
-        expect(await kvShare.purgeRevokedHandles()).toBeGreaterThanOrEqual(1);
+        expect(
+            await kvShare.purgeRevokedHandles(new AbortController().signal),
+        ).toBeGreaterThanOrEqual(1);
         expect(await exists(old)).toBe(false);
         expect(await exists(recent)).toBe(true);
         expect(await exists(live)).toBe(true);
+    });
 
-        // Another node checking the same day finds the gate held.
-        const again = await retiredAt('purge-again:', now() - 31 * DAY);
-        expect(await kvShare.purgeRevokedHandles()).toBeNull();
-        expect(await exists(again)).toBe(true);
+    it('stops purging between batches once its signal aborts', async () => {
+        const kvShare = env.server.services.kvShare;
+        const old = await retiredAt('purge-aborted:', now() - 31 * DAY);
+        const controller = new AbortController();
+        controller.abort();
+
+        expect(await kvShare.purgeRevokedHandles(controller.signal)).toBe(0);
+        expect(await exists(old)).toBe(true);
     });
 });

src/backend/stores/events/columnWidths.ts

@@ -19,8 +19,11 @@
 
 import { HttpError } from '../../core/http/HttpError.js';
 
-/** How wide one column is, and the name a caller knows the value by. */
-interface ColumnWidth {
+/**
+ * How wide one column is, and the name a caller knows the value by. Called by
+ * extensions.
+ */
+export interface ColumnWidth {
     /** Column in the migrations, so a schema change is greppable from here. */
     column: string;
     max: number;

src/backend/stores/events/types.ts

@@ -38,6 +38,20 @@ export const runsInWorker = (row: {
     handlerName?: string | null;
 }): boolean => row.appUid !== null && !!row.handlerName;
 
+/**
+ * Token, and subject, of the subscription row a recurring or `once` schedule
+ * owns. Those rows carry the schedule's lifecycle (suspend, resume, reap) and
+ * are kept out of dispatch, the durable quotas and the durable listing. Called
+ * by extensions.
+ */
+export const SCHEDULE_TOKEN_PREFIX = 'schedule:';
+
+export const scheduleToken = (scheduleUid: string): string =>
+    `${SCHEDULE_TOKEN_PREFIX}${scheduleUid}`;
+
+export const isScheduleRow = (row: { token: string }): boolean =>
+    row.token.startsWith(SCHEDULE_TOKEN_PREFIX);
+
 /**
  * What a row's delivery re-check runs against. An access mode for a filesystem
  * row, which composes with its anchor uid; the whole grant string for a row on

src/backend/stores/notification/NotificationStore.js

@@ -33,6 +33,9 @@ import { PuterStore } from '../types';
  * }} NotificationInsert
  */
 
+/** How long a probe that found no `send_key` column stands. */
+const SEND_KEY_REPROBE_MS = 60_000;
+
 /**
  * Fixed width and compared byte for byte on every engine, whatever `group`
  * holds.
@@ -75,6 +78,15 @@ const sliceWhere = (userId, { audience, appUid, excludeTypes }) => {
 };
 
 export class NotificationStore extends PuterStore {
+    #sendKeyPresent = false;
+    #sendKeyProbedAt = Number.NEGATIVE_INFINITY;
+    /** @type {Promise<void> | null} */
+    #sendKeyProbe = null;
+
+    async onServerStart() {
+        await this.#hasSendKey();
+    }
+
     // -- Reads --------------------------------------------------------
 
     /**
@@ -96,6 +108,20 @@ export class NotificationStore extends PuterStore {
         return this.#normalizeRow(rows[0]) ?? null;
     }
 
+    /**
+     * The row a delayed send wrote or last replaced, from the primary. Called
+     * by extensions.
+     *
+     * @param {string} sendKey
+     */
+    async getBySendKey(sendKey) {
+        const rows = await this.clients.db.pread(
+            'SELECT * FROM `notification` WHERE `send_key` = ? LIMIT 1',
+            [sendKey],
+        );
+        return this.#normalizeRow(rows[0]) ?? null;
+    }
+
     /**
      * The newest row `appUid` wrote for the user under `group`, or null when it
      * was shown or acknowledged: an older row is never picked up in its place.
@@ -341,6 +367,7 @@ export class NotificationStore extends PuterStore {
             type,
             group_key: key,
             replaced_at: null,
+            send_key: null,
         });
     }
 
@@ -354,6 +381,95 @@ export class NotificationStore extends PuterStore {
         return (await this.create(args)).uid;
     }
 
+    /**
+     * The statements that write a delayed send's notification, for the caller's
+     * transaction. Each runs only while `guard` holds. With `replaceUid` the
+     * open row is replaced in place and takes `sendKey`; the insert, whose uid
+     * is `sendKey`, then runs only if that row was shown or dismissed in
+     * between. Called by extensions.
+     *
+     * @param {{
+     *     sendKey: string;
+     *     userId: number;
+     *     value: unknown;
+     *     type: string;
+     *     audience: string;
+     *     appUid: string | null;
+     *     group?: string | null;
+     *     replaceUid?: string | null;
+     *     at: number;
+     * }} send
+     *   `at` is epoch ms, the row's time either way.
+     * @param {{ table: string; where: string; values: unknown[] }} guard
+     * @returns {{ statement: string; values: unknown[] }[]}
+     */
+    sendEntries(send, guard) {
+        const db = this.clients.db;
+        const serialized = JSON.stringify(send.value ?? {});
+        const seconds = Math.floor(send.at / 1000);
+        const fence = `SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where}`;
+        const entries = [];
+        if (send.replaceUid) {
+            entries.push({
+                statement:
+                    'UPDATE `notification` SET `value` = ?, `shown` = NULL, ' +
+                    `\`created_at\` = ${db.timestampFromUnixSql()}, ` +
+                    '`replaced_at` = ?, `send_key` = ? WHERE `uid` = ? ' +
+                    'AND `user_id` = ? AND `shown` IS NULL ' +
+                    `AND \`acknowledged\` IS NULL AND EXISTS (${fence})`,
+                values: [
+                    serialized,
+                    seconds,
+                    send.at,
+                    send.sendKey,
+                    send.replaceUid,
+                    send.userId,
+                    ...guard.values,
+                ],
+            });
+        }
+        // Already written: the new row's uid is the key, or a replaced row
+        // carries it. Point lookups on `uid`, since a miss on the mostly-NULL
+        // `send_key` index gap-locks where every plain insert goes.
+        const written = send.replaceUid
+            ? {
+                  sql:
+                      'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?) ' +
+                      'AND NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ? AND `send_key` = ?)',
+                  values: [send.sendKey, send.replaceUid, send.sendKey],
+              }
+            : {
+                  sql: 'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?)',
+                  values: [send.sendKey],
+              };
+        // Postgres can't type a bare parameter in a SELECT list.
+        const param = (type) =>
+            db.case({ postgres: `CAST(? AS ${type})`, otherwise: '?' });
+        entries.push({
+            statement:
+                'INSERT INTO `notification` (`uid`, `user_id`, `value`, `type`, ' +
+                '`audience`, `app_uid`, `group_key`, `created_at`, `send_key`) ' +
+                `SELECT ${param('TEXT')}, ${param('INTEGER')}, ${param('JSONB')}, ` +
+                `${param('TEXT')}, ${param('TEXT')}, ${param('TEXT')}, ` +
+                `${param('TEXT')}, ${db.timestampFromUnixSql()}, ${param('TEXT')} ` +
+                `FROM \`${guard.table}\` WHERE ${guard.where} AND ${written.sql}`,
+            values: [
+                send.sendKey,
+                send.userId,
+                serialized,
+                send.type,
+                send.audience,
+                send.appUid,
+                groupKey(send.group),
+                seconds,
+                send.sendKey,
+                ...guard.values,
+                ...written.values,
+            ],
+        });
+        return entries;
+    }
+
     /**
      * Rewrite a notification the recipient hasn't dismissed. `shown` is cleared
      * with it, so changed wording goes out again on their next connect —
@@ -362,6 +478,10 @@ export class NotificationStore extends PuterStore {
      * `onlyUnseen` also skips a row already shown. `replacedAt`, in epoch ms,
      * records the rewrite as a replacement and dates the row to it.
      *
+     * `send_key` is cleared with it whenever the column exists, on every node:
+     * the content is no longer what a delayed send wrote, so its recovery must
+     * not deliver it under that send's key.
+     *
      * False when no such row exists, so callers can fall back to a fresh
      * notification instead of dropping what they were reporting.
      *
@@ -378,6 +498,7 @@ export class NotificationStore extends PuterStore {
         const serialized =
             typeof value === 'string' ? value : JSON.stringify(value ?? {});
         const set = ['`value` = ?', '`shown` = NULL'];
+        if (await this.#hasSendKey()) set.push('`send_key` = NULL');
         const params = [serialized];
         if (replacedAt !== undefined) {
             set.push(
@@ -478,6 +599,45 @@ export class NotificationStore extends PuterStore {
 
     // -- Internals ----------------------------------------------------
 
+    /**
+     * Whether `notification.send_key` exists, asked of the primary. Present is
+     * kept for the process; absent is asked again at most once a minute, so a
+     * node started before the column was added clears it soon after.
+     *
+     * @returns {Promise<boolean>}
+     */
+    async #hasSendKey() {
+        if (this.#sendKeyPresent) return true;
+        if (Date.now() - this.#sendKeyProbedAt < SEND_KEY_REPROBE_MS)
+            return false;
+        if (this.#sendKeyProbe) {
+            await this.#sendKeyProbe;
+            return this.#sendKeyPresent;
+        }
+        const probe = this.#probeSendKey();
+        this.#sendKeyProbe = probe;
+        try {
+            await probe;
+        } finally {
+            if (this.#sendKeyProbe === probe) this.#sendKeyProbe = null;
+        }
+        return this.#sendKeyPresent;
+    }
+
+    /** Never rejects: an error reads as no column. */
+    async #probeSendKey() {
+        try {
+            await this.clients.db.pread(
+                'SELECT `send_key` FROM `notification` WHERE 1 = 0',
+                [],
+            );
+            this.#sendKeyPresent = true;
+        } catch {
+            this.#sendKeyPresent = false;
+        }
+        this.#sendKeyProbedAt = Date.now();
+    }
+
     #normalizeRow(row) {
         if (!row) return null;
         if (typeof row.value === 'string') {

src/backend/stores/notification/NotificationStore.sendKey.test.ts

@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * `send_key` is cleared on every rewrite once the column exists, whichever node
+ * rewrites and whatever its config: a row a delayed send wrote and another node
+ * replaced must not be delivered again under that send's key.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    describe,
+    expect,
+    it,
+    vi,
+} from 'vitest';
+import type { PuterServer } from '../../server.js';
+import { setupTestServer } from '../../testUtil.js';
+import type { IConfig } from '../../types.js';
+import { NotificationStore } from './NotificationStore.js';
+
+let server: PuterServer;
+let userId: number;
+
+/** A store on the same database, as another node with its own config. */
+const node = (config: Partial<IConfig>): NotificationStore =>
+    new NotificationStore(
+        config as IConfig,
+        server.clients as never,
+        server.stores as never,
+    );
+
+/** A row as a delayed send writes it, keyed `sendKey`. */
+const writeSent = async (store: NotificationStore): Promise<string> => {
+    const sendKey = uuidv4();
+    await server.clients.db.batchWrite(
+        store.sendEntries(
+            {
+                sendKey,
+                userId,
+                value: { title: 'later' },
+                type: 'app-message',
+                audience: 'account',
+                appUid: null,
+                at: Date.now(),
+            },
+            { table: 'user', where: '`id` = ?', values: [userId] },
+        ),
+    );
+    return sendKey;
+};
+
+const sendKeyOf = async (uid: string): Promise<unknown> => {
+    const [row] = await server.clients.db.pread(
+        'SELECT `send_key` FROM `notification` WHERE `uid` = ?',
+        [uid],
+    );
+    return row?.send_key;
+};
+
+beforeAll(async () => {
+    server = await setupTestServer();
+    const username = `sendkey-${Math.random().toString(36).slice(2, 10)}`;
+    const user = await server.stores.user.create({
+        username,
+        uuid: uuidv4(),
+        password: null,
+        email: `${username}@test.local`,
+    });
+    userId = user.id;
+});
+
+afterAll(async () => {
+    await server?.shutdown();
+});
+
+afterEach(() => {
+    vi.restoreAllMocks();
+});
+
+describe('clearing send_key on a rewrite', () => {
+    it('clears it from a row another node wrote, whatever either config says', async () => {
+        const writer = node({ events: { enabled: true } } as Partial<IConfig>);
+        const rewriter = node({});
+        const sendKey = await writeSent(writer);
+        expect(await sendKeyOf(sendKey)).toBe(sendKey);
+
+        await expect(
+            rewriter.updateValue(sendKey, userId, { title: 'replaced' }),
+        ).resolves.toBe(true);
+        expect(await sendKeyOf(sendKey)).toBeNull();
+    });
+
+    it('probes once and keeps a present column for the process', async () => {
+        const store = node({});
+        const pread = vi.spyOn(server.clients.db, 'pread');
+        const first = await writeSent(store);
+        const second = await writeSent(store);
+
+        await store.updateValue(first, userId, { v: 1 });
+        await store.updateValue(second, userId, { v: 2 });
+
+        const probes = pread.mock.calls.filter(([sql]) =>
+            String(sql).includes('WHERE 1 = 0'),
+        );
+        expect(probes).toHaveLength(1);
+        expect(await sendKeyOf(first)).toBeNull();
+        expect(await sendKeyOf(second)).toBeNull();
+    });
+
+    it('rewrites without naming the column while it is absent, and clears it after the next probe once it exists', async () => {
+        const db = server.clients.db;
+        const store = node({});
+        const sendKey = await writeSent(store);
+        const write = vi.spyOn(db, 'write');
+        const realPread = db.pread.bind(db);
+        let columnExists = false;
+        vi.spyOn(db, 'pread').mockImplementation(async (sql, values) => {
+            if (!columnExists && String(sql).includes('WHERE 1 = 0'))
+                throw new Error('no such column: send_key');
+            return realPread(sql, values);
+        });
+        let now = Date.now();
+        vi.spyOn(Date, 'now').mockImplementation(() => now);
+        const rewriteSql = () =>
+            String(
+                write.mock.calls.findLast(([sql]) =>
+                    String(sql).startsWith('UPDATE `notification` SET `value`'),
+                )?.[0],
+            );
+
+        await store.updateValue(sendKey, userId, { v: 1 });
+        expect(rewriteSql()).not.toContain('send_key');
+        expect(await sendKeyOf(sendKey)).toBe(sendKey);
+
+        // Within the minute the absent answer stands.
+        columnExists = true;
+        now += 30_000;
+        await store.updateValue(sendKey, userId, { v: 2 });
+        expect(rewriteSql()).not.toContain('send_key');
+        expect(await sendKeyOf(sendKey)).toBe(sendKey);
+
+        now += 31_000;
+        await store.updateValue(sendKey, userId, { v: 3 });
+        expect(rewriteSql()).toContain('`send_key` = NULL');
+        expect(await sendKeyOf(sendKey)).toBeNull();
+    });
+
+    it('shares one probe between rewrites that race it', async () => {
+        const db = server.clients.db;
+        const store = node({});
+        const keys = await Promise.all([writeSent(store), writeSent(store)]);
+        const pread = vi.spyOn(db, 'pread');
+
+        await Promise.all(
+            keys.map((uid) => store.updateValue(uid, userId, { v: 1 })),
+        );
+
+        const probes = pread.mock.calls.filter(([sql]) =>
+            String(sql).includes('WHERE 1 = 0'),
+        );
+        expect(probes).toHaveLength(1);
+        for (const uid of keys) expect(await sendKeyOf(uid)).toBeNull();
+    });
+});
+
+describe('writing a delayed send twice', () => {
+    const entriesFor = (sendKey: string, replaceUid: string | null = null) =>
+        node({}).sendEntries(
+            {
+                sendKey,
+                userId,
+                value: { title: 'later' },
+                type: 'app-message',
+                audience: 'account',
+                appUid: null,
+                replaceUid,
+                at: Date.now(),
+            },
+            { table: 'user', where: '`id` = ?', values: [userId] },
+        );
+    const rowsKeyed = async (uid: string): Promise<number> =>
+        (
+            await server.clients.db.pread(
+                'SELECT 1 FROM `notification` WHERE `uid` = ?',
+                [uid],
+            )
+        ).length;
+
+    it('writes nothing again once the row was rewritten and lost its key', async () => {
+        const sendKey = await writeSent(node({}));
+        await node({}).updateValue(sendKey, userId, { title: 'replaced' });
+        expect(await sendKeyOf(sendKey)).toBeNull();
+
+        await server.clients.db.batchWrite(entriesFor(sendKey));
+        expect(await rowsKeyed(sendKey)).toBe(1);
+    });
+
+    it('adds no row once the replaced row carries the key', async () => {
+        const replaced = await writeSent(node({}));
+        const sendKey = uuidv4();
+        await server.clients.db.batchWrite(entriesFor(sendKey, replaced));
+        expect(await sendKeyOf(replaced)).toBe(sendKey);
+        expect(await rowsKeyed(sendKey)).toBe(0);
+
+        await server.clients.db.batchWrite(entriesFor(sendKey, replaced));
+        expect(await rowsKeyed(sendKey)).toBe(0);
+    });
+});

src/backend/stores/notification/NotificationStore.test.js

@@ -346,7 +346,9 @@ describe('NotificationStore', () => {
         const uids = (rows) => rows.map((r) => r.uid).sort();
 
         expect(
-            uids(await store.listScoped(u.id, { audience: 'developer', appUid })),
+            uids(
+                await store.listScoped(u.id, { audience: 'developer', appUid }),
+            ),
         ).toEqual([mine.uid]);
         // `null` is the rows naming no app, not "any app".
         expect(

src/backend/stores/session/SessionStore.js

@@ -88,6 +88,8 @@ const predates = (row, notBefore) =>
     Number(row.created_at) < notBefore;
 
 export class SessionStore extends PuterStore {
+    /** Worker lookups under way, so a burst of first calls inserts once. */
+    #workerLookups = new Map();
     #lastSessionTouchMs = new Map();
     #lastUserTouchMs = new Map();
 
@@ -847,7 +849,21 @@ export class SessionStore extends PuterStore {
      */
     async getOrCreateWorker(userId, opts = {}) {
         if (!userId || !opts.workerName) return null;
+        const key = `${this.#cacheKeyWorker(
+            userId,
+            opts.appUid ?? null,
+            String(opts.workerName),
+        )}|${opts.notBefore ?? ''}`;
+        const running = this.#workerLookups.get(key);
+        if (running) return running;
+        const started = this.#getOrCreateWorker(userId, opts).finally(() =>
+            this.#workerLookups.delete(key),
+        );
+        this.#workerLookups.set(key, started);
+        return started;
+    }
 
+    async #getOrCreateWorker(userId, opts) {
         const existing = await this.getWorker(userId, opts);
         if (existing && !predates(existing, opts.notBefore ?? null)) {
             return existing;

src/backend/stores/session/SessionStore.test.ts

@@ -1008,6 +1008,7 @@ describe('SessionStore', () => {
             const user = await makeUser();
             const appUid = `app-${uuidv4()}`;
             const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
+            const write = vi.spyOn(server.clients.db, 'write');
             const results = await Promise.all(
                 Array.from({ length: 10 }, () =>
                     target.getOrCreateWorker(user.id, { appUid, workerName }),
@@ -1015,6 +1016,13 @@ describe('SessionStore', () => {
             );
             const uuids = new Set(results.map((r: { uuid: string }) => r.uuid));
             expect(uuids.size).toBe(1);
+            // One process inserts once, however many ask at the same time.
+            expect(
+                write.mock.calls.filter(([sql]) =>
+                    String(sql).startsWith('INSERT INTO `sessions`'),
+                ),
+            ).toHaveLength(1);
+            write.mockRestore();
 
             // And only one row ever made it into the table.
             const rows = await server.clients.db.read(
@@ -1024,6 +1032,62 @@ describe('SessionStore', () => {
             expect(rows).toHaveLength(1);
         });
 
+        it('a failed first insert rejects every racer, and the next call starts afresh', async () => {
+            const user = await makeUser();
+            const appUid = `app-${uuidv4()}`;
+            const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
+            const write = vi
+                .spyOn(server.clients.db, 'write')
+                .mockImplementationOnce(async () => {
+                    throw new Error('primary down');
+                });
+            const racers = Array.from({ length: 3 }, () =>
+                target.getOrCreateWorker(user.id, { appUid, workerName }),
+            );
+            for (const racer of racers)
+                await expect(racer).rejects.toThrow('primary down');
+            write.mockRestore();
+            const row = await target.getOrCreateWorker(user.id, {
+                appUid,
+                workerName,
+            });
+            expect(row.kind).toBe('worker');
+        });
+
+        it('lookups for different apps do not wait on each other', async () => {
+            const user = await makeUser();
+            const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
+            const db = server.clients.db;
+            const original = db.write.bind(db);
+            let release!: () => void;
+            const held = new Promise<void>((resolve) => (release = resolve));
+            // The first app's insert hangs until released.
+            const write = vi
+                .spyOn(db, 'write')
+                .mockImplementationOnce(async (...args) => {
+                    await held;
+                    return original(...(args as [string, unknown[]]));
+                });
+            const slow = target.getOrCreateWorker(user.id, {
+                appUid: `app-${uuidv4()}`,
+                workerName,
+            });
+            const other = await Promise.race([
+                target.getOrCreateWorker(user.id, {
+                    appUid: `app-${uuidv4()}`,
+                    workerName,
+                }),
+                new Promise((resolve) =>
+                    setTimeout(() => resolve('stuck'), 1500),
+                ),
+            ]);
+            expect(other).not.toBe('stuck');
+            release();
+            const slowRow = await slow;
+            expect(slowRow.uuid).not.toBe((other as { uuid: string }).uuid);
+            write.mockRestore();
+        });
+
         it('returns null when called with falsy inputs', async () => {
             expect(
                 await target.getOrCreateWorker(null, { workerName: 'wk' }),
@@ -1094,8 +1158,9 @@ describe('SessionStore', () => {
             const uuids = rows.map((row: { uuid: string }) => row.uuid);
             expect(uuids).toContain(a.uuid);
             expect(uuids).toContain(b.uuid);
-            expect(rows.every((row: { id: number }) => Number.isFinite(row.id)))
-                .toBe(true);
+            expect(
+                rows.every((row: { id: number }) => Number.isFinite(row.id)),
+            ).toBe(true);
         });
 
         it('does not return a revoked session', async () => {

src/backend/testUtil.ts

@@ -152,6 +152,8 @@ export const setupTestServer = async (
             database: testDatabaseDefault(),
             dynamo: { inMemory: true, bootstrapTables: true },
             redis: { useMock: true },
+            // Suites run system jobs themselves, not on the poll timer.
+            jobs: { enabled: false },
             s3: { localConfig: { inMemory: true } },
             no_default_user: true,
             no_devwatch: true,

src/backend/types.ts

@@ -256,7 +256,12 @@ export interface IPreludeConfig {
      * an RCS agent provisioned in the Prelude account to actually use RCS.
      */
     preferredChannel?:
-        'sms' | 'rcs' | 'whatsapp' | 'viber' | 'zalo' | 'telegram';
+        | 'sms'
+        | 'rcs'
+        | 'whatsapp'
+        | 'viber'
+        | 'zalo'
+        | 'telegram';
 }
 
 /**
@@ -642,6 +647,21 @@ export interface IDevWatcherConfig {
     }>;
 }
 
+/**
+ * System jobs config (`jobs`). An extension that runs jobs types its own keys
+ * by augmenting this interface.
+ *
+ * - `namespace` — keys this deployment's jobs and schedules, so deployments
+ *   sharing one database or Redis never run each other's. Defaults to `domain`;
+ *   one that can't be held falls back to `default` (`resolveJobsNamespace`).
+ * - `enabled` — `false` keeps this node from polling for jobs (logged at start);
+ *   the rest of the fleet still runs them. Test servers turn it off.
+ */
+export interface IJobsConfig {
+    namespace?: string;
+    enabled?: boolean;
+}
+
 /**
  * Root config. Only `port` and `extensions` are mandatory; they are added in
  * `IConfig` below. Every config field is declared here with a doc comment so
@@ -1270,6 +1290,16 @@ interface IConfigOptional {
         forwardSession?: boolean;
         /** How long a handler has to answer an invocation. Default 30 s. */
         invokeTimeoutMs?: number;
+        /**
+         * Invocations sent to the dispatcher in one request. Default 60; 1
+         * sends each on its own.
+         */
+        invokeBatchSize?: number;
+        /**
+         * How long the first invocation of a batch waits for others to join it.
+         * Default 5 ms.
+         */
+        invokeBatchWindowMs?: number;
         /**
          * Dispatch namespace events workers are deployed into. Must not be the
          * one the public worker dispatcher serves — that namespace is reachable
@@ -1291,6 +1321,9 @@ interface IConfigOptional {
         internalSecret?: string;
     };
 
+    /** System jobs (`services.jobs`); see {@link IJobsConfig}. */
+    jobs?: IJobsConfig;
+
     /**
      * Key-value sharing beyond watching.
      *
@@ -1346,8 +1379,7 @@ export interface ServerDrainInfo {
 }
 
 export interface WithCostsReporting extends WithLifecycle {
-    getReportedCosts?: () =>
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    getReportedCosts?: () => // eslint-disable-next-line @typescript-eslint/no-explicit-any
         | Promise<Record<string, any>[]>
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         | Record<string, any>[];

src/backend/util/jobsNamespace.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import type { IConfig } from '../types.js';
+import { resolveJobsNamespace } from './jobsNamespace.js';
+
+const config = (partial: Partial<IConfig>): IConfig => partial as IConfig;
+
+describe('resolveJobsNamespace', () => {
+    afterEach(() => {
+        vi.restoreAllMocks();
+    });
+
+    it('takes jobs.namespace first, then the domain, then "default"', () => {
+        expect(
+            resolveJobsNamespace(
+                config({ jobs: { namespace: 'ns-1' }, domain: 'puter.test' }),
+            ),
+        ).toBe('ns-1');
+        expect(resolveJobsNamespace(config({ domain: 'puter.test' }))).toBe(
+            'puter.test',
+        );
+        expect(resolveJobsNamespace(config({}))).toBe('default');
+    });
+
+    it('falls back to "default" for a domain with a port, logging once', () => {
+        const error = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const withPort = config({ domain: 'puter.localhost:4100' });
+        expect(resolveJobsNamespace(withPort)).toBe('default');
+        expect(resolveJobsNamespace(withPort)).toBe('default');
+        expect(error).toHaveBeenCalledTimes(1);
+        expect(String(error.mock.calls[0][0])).toContain(
+            'puter.localhost:4100',
+        );
+    });
+
+    it('falls back to "default" for an invalid jobs.namespace', () => {
+        const error = vi.spyOn(console, 'error').mockImplementation(() => {});
+        expect(
+            resolveJobsNamespace(
+                config({ jobs: { namespace: 'a:b' }, domain: 'puter.test' }),
+            ),
+        ).toBe('default');
+        expect(resolveJobsNamespace(config({ jobs: { namespace: '' } }))).toBe(
+            'default',
+        );
+        expect(
+            resolveJobsNamespace(
+                config({ jobs: { namespace: 'x'.repeat(129) } }),
+            ),
+        ).toBe('default');
+        expect(error).toHaveBeenCalledTimes(3);
+    });
+});

src/backend/util/jobsNamespace.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type { IConfig } from '../types.js';
+
+/** A namespace must fit the jobs and schedules tables and a Redis key segment. */
+const JOBS_NAMESPACE_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+const warned = new Set<string>();
+
+/**
+ * The namespace system jobs and schedules are keyed by, so deployments sharing
+ * a database or Redis stay apart: `jobs.namespace`, else `domain`, else
+ * `default`. One that can't be held (a domain with a port, say) falls back to
+ * `default`, with one logged error per value.
+ */
+export const resolveJobsNamespace = (config: IConfig): string => {
+    const configured = config.jobs?.namespace ?? config.domain ?? 'default';
+    if (JOBS_NAMESPACE_PATTERN.test(configured)) return configured;
+    if (!warned.has(configured)) {
+        warned.add(configured);
+        console.error(
+            `[jobs] namespace "${configured}" (jobs.namespace, else domain) must match ${JOBS_NAMESPACE_PATTERN}; using "default". Set jobs.namespace to keep deployments sharing a database apart.`,
+        );
+    }
+    return 'default';
+};

src/backend/util/retryBackoff.test.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+    deliveryBackoffMs,
+    EVENTS_RETRY_BASE_MS,
+    EVENTS_RETRY_MAX_MS,
+} from './retryBackoff.js';
+
+describe('deliveryBackoffMs', () => {
+    it('doubles from the base per failure and stops at the ceiling', () => {
+        expect(deliveryBackoffMs(0)).toBe(EVENTS_RETRY_BASE_MS);
+        expect(deliveryBackoffMs(1)).toBe(EVENTS_RETRY_BASE_MS);
+        expect(deliveryBackoffMs(3)).toBe(EVENTS_RETRY_BASE_MS * 4);
+        expect(deliveryBackoffMs(50)).toBe(EVENTS_RETRY_MAX_MS);
+    });
+});

src/backend/util/retryBackoff.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// A retry on a fixed cadence turns one broken dependency into a permanent load
+// on it, so the wait doubles per attempt up to a ceiling.
+
+/** Wait before the first retry. */
+export const EVENTS_RETRY_BASE_MS = 2_000;
+
+/** Longest wait between retries, however many have failed. */
+export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
+
+/** How long to wait after `attempts` failures in a row. */
+export const deliveryBackoffMs = (attempts: number): number =>
+    Math.min(
+        EVENTS_RETRY_MAX_MS,
+        EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
+    );

src/docs/src/Events.md

@@ -25,7 +25,17 @@ What you watch is written as a subject string:
 
 [Subjects and events](/Events/subjects/) has the full subject syntax and the shape of each event.
 
-With the [User-Pays Model](/user-pays-model/), deliveries are billed to the user holding the subscription, not to you. See [Rate Limits and Quotas](/rate-limits-and-quotas/#events) for limits and costs.
+To run a handler on a timer instead of when data changes, create a schedule with [`onSchedule()`](/Events/onSchedule/). It runs your handler at an interval, on a cron expression in the user's time zone, or once at a set time, even while your app is closed:
+
+```js
+await puter.events.onSchedule({
+    cron: '0 9 * * 1-5',
+    tz: 'America/New_York',
+    handlerName: 'sendDigest',
+});
+```
+
+With the [User-Pays Model](/user-pays-model/), deliveries and schedule runs are billed to the user holding the subscription or schedule, not to you. See [Rate Limits and Quotas](/rate-limits-and-quotas/#events) for limits and costs.
 
 ## Features
 
@@ -247,6 +257,10 @@ These Events features are supported out of the box when using Puter.js:
 - **[`puter.events.list()`](/Events/list/)** - List the persistent subscriptions you hold
 - **[`puter.events.update()`](/Events/update/)** - Change a persistent subscription's context without ending it
 - **[`puter.events.unsubscribe()`](/Events/unsubscribe/)** - End a persistent subscription
+- **[`puter.events.onSchedule()`](/Events/onSchedule/)** - Run a handler at an interval, on a cron expression, or once at a set time
+- **[`puter.events.listSchedules()`](/Events/listSchedules/)** - List the schedules you hold
+- **[`puter.events.getSchedule()`](/Events/getSchedule/)** - Read one schedule
+- **[`puter.events.removeSchedule()`](/Events/removeSchedule/)** - End a schedule
 - **[`puter.events.fetch()`](/Events/fetch/)** - Read the notifications you missed while nothing was listening
 - **[`puter.events.handlers`](/Events/handlers/)** - Publish, list and remove the named handlers persistent subscriptions run
 - **[`puter.events.workers`](/Events/workers/)** - List and destroy an app's events worker
@@ -266,6 +280,10 @@ You can see various Puter.js Events features in action from the following exampl
   - [Find Suspended Subscriptions](/playground/events-list-suspended/)
   - [Change a Subscription's Context](/playground/events-update/)
   - [End a Persistent Subscription](/playground/events-unsubscribe/)
+- Schedules
+  - [List Schedules](/playground/events-list-schedules/)
+  - [Read a Schedule](/playground/events-get-schedule/)
+  - [Remove a Schedule](/playground/events-remove-schedule/)
 - Catching up
   - [Catch Up on Notifications](/playground/events-fetch/)
   - [Catch Up, Then Keep Listening](/playground/events-fetch-then-listen/)

src/docs/src/Events/getSchedule.md

@@ -0,0 +1,67 @@
+---
+title: puter.events.getSchedule()
+description: Read one schedule, including one that has ended.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+Reads one schedule created with [`puter.events.onSchedule()`](/Events/onSchedule/), or one notification your app stored to send later with [`puter.notifications.notify()`](/Notifications/notify/#send-it-later). An ended schedule can be read for 7 days after it ends.
+
+An app can read only the schedules it created. An account session can read any of the user's schedules. An id this caller can't read reads as not existing, so the call doesn't reveal which schedules exist.
+
+On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.getSchedule(scheduleId)
+```
+
+## Parameters
+
+#### `scheduleId` (String) (required)
+The `scheduleId` from [`onSchedule()`](/Events/onSchedule/), [`notify()`](/Notifications/notify/#send-it-later) or [`listSchedules()`](/Events/listSchedules/).
+
+## Return value
+
+A `Promise` that resolves to the schedule, the object [`onSchedule()`](/Events/onSchedule/#return-value) returns, without `off()`. Its `state` says whether it's `active`, `suspended`, `sending` or `ended`, and `nextRunAt` says when it's next due.
+
+The promise rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `events_schedule_not_found` | No such schedule, or it isn't this caller's. Also for an empty `scheduleId`. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `too_many_requests` | Over the listing rate limit, which is shared with [`listSchedules()`](/Events/listSchedules/). |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">Check when a stored notification goes out</strong>
+
+```html;events-get-schedule
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            const later = await puter.notifications.notify({
+                title: 'Stand-up in 10 minutes',
+                sendAt: Date.now() + 10 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            const schedule = await puter.events.getSchedule(later.scheduleId);
+            const due = schedule.nextRunAt ? new Date(schedule.nextRunAt).toLocaleTimeString() : 'nothing due';
+            puter.print(`${schedule.state}, due ${due}<br>`);
+
+            // Ended schedules can still be read
+            await puter.notifications.cancel(later.scheduleId);
+            const ended = await puter.events.getSchedule(later.scheduleId);
+            puter.print(`${ended.state} (${ended.endReason})<br>`);
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Events/handlers.md

@@ -1,12 +1,12 @@
 ---
 title: puter.events.handlers
-description: Publish, list and remove the named handlers a persistent subscription runs.
+description: Publish, list and remove the named handlers that persistent subscriptions and schedules run.
 platforms: [websites, apps, nodejs, workers]
 ---
 
 <div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
 
-A **handler** is a function your app publishes once, under a name, for persistent subscriptions to run. A handler runs only when a subscription bound to its name has a delivery. Nothing else can trigger it.
+A **handler** is a function your app publishes once, under a name, for [persistent subscriptions](/Events/onPersistent/) and [schedules](/Events/onSchedule/) to run. A handler runs only when a subscription bound to its name has a delivery, or a schedule bound to it is due. Nothing else can trigger it.
 
 Only the app's owner can publish, list, and remove its handlers. Inside an app, including a website using Puter.js, these calls act on that app, and work only while the signed-in user owns it. Outside an app, such as in Node.js with your account token, name the app with `appUid`.
 
@@ -40,6 +40,17 @@ await puter.events.onPersistent({ subject: 'fs:~/inbox', handlerName: 'ingestUpl
 
 An identifier the check can't resolve is refused when you publish, instead of failing on the first delivery.
 
+## What a handler is called with
+
+A handler gets one object. Which `event` it carries depends on what ran it:
+
+- **A subscription delivery** carries the change that happened, in the shape [Subjects and events](/Events/subjects/#the-event) describes, or a [gap marker](/Events/subjects/#gaps).
+- **A schedule run** carries `op: 'tick'`, the `scheduleId`, and the `scheduledFor` time the run is for, as [`onSchedule()`](/Events/onSchedule/#the-event) describes.
+
+The other bindings are the same for both: `ctx`, `user` and `fetch`, plus `ack` on a `queue` subscription. See [Where the handler runs](/Events/onPersistent/#where-the-handler-runs). One handler can serve both, and tell them apart by `event.op`.
+
+Schedule runs and `queue` deliveries are at-least-once. A retry, or a handler still running past its 30 seconds when the retry starts, can run the same event twice, and even at the same time. Both attempts have the same `event.id`, so record it once the work is done and skip ids you've already handled.
+
 ## `publish()`
 
 ```js
@@ -78,7 +89,7 @@ puter.events.handlers.list()
 puter.events.handlers.list(options)
 ```
 
-Resolves to `[{ name, hash, updatedAt, subscriptions }]` for every handler the app has published, sorted by name. Each `subscriptions` count includes the suspended subscriptions bound to that name. The `options.appUid` option works as it does for `publish()`.
+Resolves to `[{ name, hash, updatedAt, subscriptions }]` for every handler the app has published, sorted by name. Each `subscriptions` count includes the subscriptions and schedules bound to that name, suspended ones included. The `options.appUid` option works as it does for `publish()`.
 
 The source is never returned.
 
@@ -93,12 +104,12 @@ Resolves to `{ name, removed, suspended }`, where `removed` is `false` when noth
 
 | Situation | What happens |
 | --- | --- |
-| No subscriptions use the name | The handler is deleted. |
-| Subscriptions use it | The handler is deleted, and its subscriptions are suspended with `suspendedReason: 'handler_not_found'`, not deleted. The app's developer gets an `app.events.suspended` notification. |
+| No subscriptions or schedules use the name | The handler is deleted. |
+| Subscriptions or schedules use it | The handler is deleted, and its subscriptions and schedules are suspended with `suspendedReason: 'handler_not_found'`, not deleted. The app's developer gets an `app.events.suspended` notification. |
 
-Publishing the name again resumes them, with the same ids and context. This is how you recover from a bad deploy.
+Publishing the name again resumes them, with the same ids and context. This is how you recover from a bad deploy. A resumed schedule runs at its next slot, and doesn't make up the runs it missed.
 
-To rename a handler, publish the new name and remove the old one. Subscriptions don't follow the rename. Users have to subscribe again, so their subscriptions never switch to different code without them knowing.
+To rename a handler, publish the new name and remove the old one. Subscriptions and schedules don't follow the rename. Users have to subscribe again, so their subscriptions never switch to different code without them knowing.
 
 An app's first published handler creates its [events worker](/Events/workers/), and removing the last one takes it down.
 

src/docs/src/Events/listSchedules.md

@@ -0,0 +1,88 @@
+---
+title: puter.events.listSchedules()
+description: List the schedules this caller holds, one page at a time.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+Lists the schedules created with [`puter.events.onSchedule()`](/Events/onSchedule/), and the notifications your app stored to send later with [`puter.notifications.notify()`](/Notifications/notify/#send-it-later). Ended schedules stay in the list for 7 days, with the reason they ended.
+
+An app sees only the schedules it created for the user it runs for. An account session, such as the Puter desktop, sees all of the user's schedules, including ones left by apps they no longer use, so that's where to clean up stray schedules. A scoped access token, such as the one in a [`getReadURL()`](/FS/getReadURL/) URL, sees none.
+
+Schedules aren't listed by [`puter.events.list()`](/Events/list/), which lists persistent subscriptions only.
+
+On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.listSchedules()
+puter.events.listSchedules(options)
+```
+
+## Parameters
+
+#### `options` (Object) (optional)
+
+- `limit` (Number): Maximum schedules per page. Capped at 200, and defaults to 50.
+- `cursor` (String | null): The `cursor` from the previous page. Leave it out, or pass `null`, for the first page.
+- `includeTotal` (Boolean): Adds `total`, the number of schedules across every page. Ask for it on the first page only.
+
+## Return value
+
+A `Promise` for one page, `{ items, cursor?, total? }`:
+
+- `items` (Array): The schedules on this page, oldest first. Each one is the object [`onSchedule()`](/Events/onSchedule/#return-value) returns, without `off()`.
+- `cursor` (String): Present while there are more pages to load. Pass it as `cursor` to get the next page.
+- `total` (Number): The number of schedules across every page, only with `includeTotal`.
+
+Pages can be shorter than `limit` before the last one, so keep going until `cursor` is absent.
+
+The `context` a schedule was created with is never returned, since it often holds secrets. The `contextKeys` field lists its key names, and `contextHash` changes whenever a value does.
+
+The promise rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `invalid_request` | `includeTotal` isn't a boolean. |
+| `bad_request` | `limit` isn't a positive number. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `too_many_requests` | Over the listing rate limit, which is shared with [`getSchedule()`](/Events/getSchedule/) and the subscription listings. |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">List every schedule, page by page</strong>
+
+```html;events-list-schedules
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            // (1) Something to list: a notification to send in an hour
+            const later = await puter.notifications.notify({
+                title: 'Time for a break',
+                sendAt: Date.now() + 60 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            // (2) Page through everything this app holds for the user
+            let cursor = null;
+            do {
+                const page = await puter.events.listSchedules({ cursor, limit: 50 });
+                for (const schedule of page.items) {
+                    const next = schedule.nextRunAt ? new Date(schedule.nextRunAt).toLocaleString() : 'nothing due';
+                    puter.print(`${schedule.kind} ${schedule.state}, next: ${next}<br>`);
+                }
+                cursor = page.cursor ?? null;
+            } while (cursor);
+
+            // (3) Cleanup
+            await puter.notifications.cancel(later.scheduleId);
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Events/onSchedule.md

@@ -0,0 +1,316 @@
+---
+title: puter.events.onSchedule()
+description: Run your app's handler at an interval, on a cron expression in a time zone, or once at a set time, even while your app is closed.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+<div class="info">Schedules are available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+Runs a handler your app published with [`puter.events.handlers.publish()`](/Events/handlers/) on a timer. A schedule can run at an interval, on a cron expression in a time zone, or once at a set time. It's stored on the user's account and runs in your app's [events worker](/Events/workers/), whether or not your app is open. It never runs in the client that created it.
+
+To run a handler when the user's data changes instead, use [`puter.events.onPersistent()`](/Events/onPersistent/).
+
+[Change Data at a Set Time](/recipes/change-data-at-a-set-time/) walks through a one-time schedule with `at`, including how to cancel and move it.
+
+Only an app can create a schedule, meaning an app running on Puter or a website using Puter.js. The schedule belongs to the user the app runs for, and to that app. On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.onSchedule(options)
+```
+
+## Parameters
+
+#### `options` (Object) (required)
+
+Give exactly one of `every`, `cron` or `at`.
+
+- `every` (Number | String): Run at this interval, in milliseconds or as a duration such as `'30m'`, `'6h'` or `'1d'`. The units are `s`, `m`, `h`, `d` and `w`. The interval must be a whole number of seconds, at least your plan's [minimum interval](#how-often-a-schedule-can-run), and at most 366 days.
+- `cron` (String): Run on a [cron expression](#cron-expressions), such as `'0 9 * * 1-5'` for 09:00 on weekdays.
+- `tz` (String): The IANA time zone the `cron` expression runs in, such as `'America/New_York'`. Defaults to `'UTC'`. Only allowed with `cron`. See [Time zones and daylight saving time](#time-zones-and-daylight-saving-time).
+- `at` (Number | String | Date): Run once, at this [time](#times). It must be at least 1 second and at most 366 days ahead. A schedule with `at` takes no `startAt`, `expiresAt` or `maxRuns`.
+- `startAt` (Number | String | Date): For `every`, the first run. Without it, the first run is one interval after this call. A `startAt` in the past sets where the intervals line up, and the first run is the next one after this call. For example, `every: '1d'` with `startAt: '2026-01-01'` runs at midnight UTC every day. For `cron`, the earliest time the schedule may run. At most 366 days ahead.
+- `expiresAt` (Number | String | Date): The last time the schedule may run. A run due at exactly this time still runs.
+- `maxRuns` (Number): How many slots the schedule has, from 1 to 100,000. It ends at its `maxRuns`-th slot counted from the first. It counts slots, not runs that happened, so slots skipped while the schedule was behind or suspended (see [When a schedule runs](#when-a-schedule-runs)) still count, and the retries of one slot count once. With `expiresAt` too, the one that comes first ends the schedule.
+- `handlerName` (String) (required): The published handler each run calls.
+- `handler` (Function | String | Object): The handler source you wrote this schedule against, as a function, a source string, or `{ file: '~/AppData/…/handler.js' }`. Only its hash is sent. The schedule is created only if it matches what's published under `handlerName`. It never runs in this client.
+- `handlerHash` (String): The SHA-256 hex digest of the handler source, in place of `handler`.
+- `context` (Object): Values the handler needs, passed to every run as a frozen `ctx`. Up to 4 KB serialized. It's read once, at this call, and can't be changed later. To change it, remove the schedule and create a new one.
+
+### Times
+
+The `at`, `startAt` and `expiresAt` options each take one of these:
+
+- An ISO-8601 string with a zone, such as `'2026-11-01T18:30:00Z'` or `'2026-11-01T18:30+02:00'`. Seconds and up to 3 fraction digits are optional.
+- A date alone, such as `'2026-11-01'`, which is read as midnight UTC.
+- A `Date`.
+- A whole number of milliseconds since the epoch.
+
+Times are rounded up to the next whole second. A string with a time and no zone, such as `'2026-11-01T18:30'`, is refused with `events_schedule_invalid`, because it could be read in any zone.
+
+These times are in milliseconds, unlike `expiresAt` on [`onPersistent()`](/Events/onPersistent/), which takes seconds. Every time a schedule reports, including `event.scheduledFor`, is in milliseconds too.
+
+## Cron expressions
+
+A `cron` expression has the 5 standard fields, separated by spaces:
+
+| Field | Values |
+| --- | --- |
+| Minute | `0`-`59` |
+| Hour | `0`-`23` |
+| Day of the month | `1`-`31` |
+| Month | `1`-`12`, or `JAN`-`DEC` |
+| Day of the week | `0`-`7`, where `0` and `7` are Sunday, or `SUN`-`SAT` |
+
+Each field takes a number, `*` for every value, a list such as `1,15`, a range such as `MON-FRI`, or a step such as `*/15` or `0-30/10`. Names can be in any case.
+
+The expression can also be one of these nicknames:
+
+| Nickname | Runs |
+| --- | --- |
+| `@yearly`, `@annually` | `0 0 1 1 *`, midnight on January 1 |
+| `@monthly` | `0 0 1 * *`, midnight on the first of the month |
+| `@weekly` | `0 0 * * 0`, midnight on Sunday |
+| `@daily`, `@midnight` | `0 0 * * *`, every midnight |
+| `@hourly` | `0 * * * *`, the start of every hour |
+
+Some cron syntax isn't supported, and is refused with `events_schedule_invalid`:
+
+- A seconds or year field, so 6 or 7 fields.
+- `?`, `L`, `W` and `#`, and a day of the week starting with `+`.
+- `@reboot`.
+- An expression over 128 characters.
+
+When neither the day of the month nor the day of the week is `*`, a day matches if either one matches, as in standard cron. So `0 9 13 * 5` runs at 09:00 on the 13th of each month and at 09:00 every Friday. A day that a month doesn't have is skipped, so `0 0 31 * *` runs only in months with 31 days.
+
+The expression has to run within 366 days of the call. An expression that never runs, such as `0 0 31 2 *`, is refused with `events_schedule_never_fires`. So is `0 0 29 2 *` while the next February 29 is more than 366 days away.
+
+### Time zones and daylight saving time
+
+A `cron` expression runs in `tz`. For example, `0 9 * * 1-5` with `tz: 'America/New_York'` runs at 09:00 New York time all year, which is 13:00 UTC in the summer and 14:00 UTC in the winter.
+
+The `tz` option takes an IANA name. A UTC offset such as `'+02:00'` is refused with `events_schedule_invalid`, since an offset has no daylight saving rules. The schedule reports the zone under its canonical name, so `'US/Eastern'` comes back as `'America/New_York'`.
+
+When the clocks change:
+
+- An expression with one minute and one hour, such as `30 2 * * *`, runs once that day. When its time is skipped because the clocks go forward, it runs at the first minute after the gap (03:00 in New York). When its time happens twice because the clocks go back, it runs once.
+- An expression with a list, range, step or `*` in the minute or hour field, such as `0,30 1 * * *` or `0 * * * *`, skips the runs that fall in the gap when the clocks go forward. When the clocks go back, it runs at each repeated time twice. Each of those runs has its own `scheduledFor` and `event.id`.
+
+## How often a schedule can run
+
+A schedule can't run more often than your plan's minimum interval, which is 1 minute on a paid plan and 15 minutes on a free one. For `every`, the interval is what's checked. For `cron`, it's the smallest gap between two runs, checked over the next 10,000 runs or 400 days, whichever ends first. For example, `0-4 9 * * *` runs at 09:00, 09:01, 09:02, 09:03 and 09:04 each day. Those runs are 1 minute apart, so a free plan can't use it. A schedule under the minimum is refused with `events_schedule_too_frequent`.
+
+The plan also limits how many schedules the user can hold for your app and in total, and how many runs per hour all of them can add up to. Over those limits, the call fails with `events_schedule_limit` or `events_schedule_budget`. With `maxRuns` on a `cron` schedule, the last run has to fall inside those same 10,000 runs or 400 days, or the call fails with `events_schedule_invalid`. See [Rate Limits and Quotas](/rate-limits-and-quotas/#schedules) for the numbers.
+
+The plan's limits are checked when a schedule is created. A schedule keeps running if the user later moves to a plan with lower limits. Schedules created after that are held to the new plan.
+
+## When a schedule runs
+
+Each run is due at a time, called its slot, which the handler gets as `event.scheduledFor`. A run usually starts within 60 seconds of its slot.
+
+So that schedules due at the same moment don't all start at once, Puter starts each schedule's runs a fixed delay after their slots. The delay comes from the schedule's id, and is a whole number of seconds under 30 seconds or a tenth of the schedule's smallest gap, whichever is smaller. For an `at` schedule it's under 10 seconds. So a schedule that runs every minute has a delay of at most 5 seconds.
+
+A run can start later than 60 seconds after its slot when it's being retried, when your app's schedules for this user go over 200 runs a minute, while your events worker is being deployed, or while the user's account is suspended. The handler can compare `event.firedAt` with `event.scheduledFor` to see how late a run is.
+
+Puter starts one attempt at a time for each schedule, and the next slot waits until the current run finishes or is given up. A handler that's still running when its 30 seconds are up can overlap its own retry, which has the same `event.id`.
+
+When a schedule falls behind, for example while a run was being retried, it doesn't make up every slot it missed. It runs the latest slot that's due, once, and `event.missed` counts the slots it skipped. A schedule that resumes after a suspension waits for its next slot, and that run's `missed` counts the slots it skipped while it was suspended.
+
+## Delivery guarantees
+
+Runs are at-least-once. A retry, or a handler that outlives its timeout, can run one slot twice. Every attempt at one slot has the same `event.id`, `<scheduleId>:<scheduledFor>`, so record it once the work is done and skip ids you've already handled:
+
+```js
+await puter.events.handlers.publish('sendDigest', async ({ event, ctx, user }) => {
+    const doneKey = `digest:${event.id}`;
+    if (await user.kv.get(doneKey)) return;
+
+    await fetch(ctx.webhookUrl, { method: 'POST', body: `Digest for ${event.scheduledFor}` });
+
+    await user.kv.set(doneKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}, { appUid });
+```
+
+## The event
+
+The handler is called with `event`, `ctx`, `user` and `fetch`, the same bindings a [persistent subscription's](/Events/onPersistent/#where-the-handler-runs) handler gets in the events worker. The `ctx` binding is the frozen `context`. There's no `ack`.
+
+A run's `event` carries:
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `subject` | String | `schedule:<scheduleId>`. |
+| `op` | String | Always `'tick'`. |
+| `id` | String | `<scheduleId>:<scheduledFor>`. The same on every attempt at one run, so use it to skip work you've already done. |
+| `scheduleId` | String | The schedule this run belongs to. |
+| `scheduledFor` | Number | The slot this run is for, in milliseconds since the epoch. |
+| `firedAt` | Number | When this attempt was sent to the handler, in milliseconds since the epoch. |
+| `missed` | Number | Slots skipped since the last run, for example while the schedule was suspended. Only this run happens for them. |
+| `missedCapped` | Boolean | Present, and `true`, when `missed` stopped counting at 1,000. |
+| `attempt` | Number | Attempts at this run, this one included. `1` on the first try. |
+
+The `user` binding is valid for 15 minutes, so use it only during the run. Writes made through `user` count toward the same [handler chain](/Events/onPersistent/#handlers-that-trigger-handlers) limit as a subscription's, starting from the depth of the call that created the schedule.
+
+## What happens when a handler fails
+
+| The handler | Result |
+| --- | --- |
+| Resolves | Done. The schedule moves on to its next slot. |
+| Throws | The run is dropped and the schedule moves on to its next slot. It counts as a failure. |
+| Throws a temporary error, meaning a network error, a timeout, a rate limit (`429`), a `5xx`, or any error with `retry: true` | Retried after 2 seconds, then 4, 8 and so on, up to 5 minutes apart, with the same `event.id`. It counts as a failure. |
+| Runs past 30 seconds | Retried on the same schedule. It counts as a failure. |
+
+A run that fails because of Puter doesn't count as a failure. It's retried on the same schedule, and waits longer if Puter's answer asked for a longer wait, up to 5 minutes. To drop the run in that case anyway, throw an error with `terminal: true`. The rules are the same as for a [`queue` subscription](/Events/onPersistent/#what-happens-when-a-handler-fails), except that the retries stop once the schedule's next run is due, or after 24 hours for an `at` schedule or a last run. The run is then dropped without billing.
+
+Five failures in a row suspend the schedule with `failures`. The developer is notified, and publishing the handler again resumes it.
+
+## Background runs need consent
+
+A schedule runs your handler while the user isn't there, so it needs the per-app permission `events:background`, as [`onPersistent()`](/Events/onPersistent/#background-delivery-needs-consent) does. Without it, the call fails with `events_background_consent_required`. To ask for it, call [`puter.perms.request()`](/Perms/request/) with the `'backgroundEvents'` resource:
+
+```js
+await puter.perms.request('backgroundEvents');
+```
+
+When the user revokes it, every schedule your app holds for them ends with `permission_revoked`. Granting it again doesn't bring them back, so create them again.
+
+## Suspended and ended schedules
+
+A schedule can stop running without being removed. It's then suspended, and [`listSchedules()`](/Events/listSchedules/) shows `state: 'suspended'` and a `suspendedReason`:
+
+| `suspendedReason` | Cause | Resumes when |
+| --- | --- | --- |
+| `handler_not_found` | Its handler was removed, or the app's [events worker](/Events/workers/) was destroyed. | A handler is published under that name again. |
+| `failures` | Its handler failed 5 times in a row. | The handler is published again. |
+| `no_credit` | The user ran out of credit. | The balance is topped up (checked every few minutes). |
+
+A suspended schedule isn't billed. One that stays suspended for 30 days ends.
+
+An ended schedule never runs again. It has `state: 'ended'` and an `endReason`, and stays in [`listSchedules()`](/Events/listSchedules/) for 7 days, after which it is deleted:
+
+| `endReason` | Cause |
+| --- | --- |
+| `removed` | It was removed with [`removeSchedule()`](/Events/removeSchedule/) or `off()`. |
+| `ended` | Its last run is done (an `at` schedule, `expiresAt` or `maxRuns`), or it stayed suspended for 30 days. |
+| `permission_revoked` | The user revoked `events:background` for your app. |
+| `uninstalled` | The user uninstalled your app. |
+| `app_deleted` | The app was deleted. |
+| `holder_gone`, `app_gone` | The user's account or the app no longer exists. |
+| `orphaned` | A create or remove that failed partway left it incomplete, and Puter cleaned it up. |
+| `delivered`, `dropped` | A notification sent later with [`notify()`](/Notifications/notify/#send-it-later) was sent, or couldn't be. |
+
+When schedules end with `permission_revoked`, the user's app gets an `app.events.ended` notification about them, which you can read with [`puter.events.fetch()`](/Events/fetch/).
+
+## Return value
+
+A `Promise` that resolves to the schedule. All times are in milliseconds since the epoch.
+
+- `scheduleId` (String): Its id, which [`getSchedule()`](/Events/getSchedule/) and [`removeSchedule()`](/Events/removeSchedule/) take.
+- `kind` (String): `'recurring'` for `every` and `cron`, `'once'` for `at`, and `'deferred'` for a notification sent later with [`notify()`](/Notifications/notify/#send-it-later).
+- `appUid` (String): The app that created it.
+- `handlerName` (String | null): The handler each run calls, or `null` once the schedule has ended.
+- `contextKeys` (Array | null), `contextHash` (String | null): The key names and a hash of the stored context, never its values. The `contextKeys` field is `null` when there's no context, and once the schedule has ended.
+- `every` (Number | null): The interval, in milliseconds, of an `every` schedule.
+- `cron` (String | null): The expression of a `cron` schedule.
+- `tz` (String): The time zone of a `cron` schedule, and `'UTC'` for the others.
+- `startAt` (Number): The first time it may run. For `at`, its only run.
+- `endsAt` (Number | null): The last time it may run, from `expiresAt` or `maxRuns`. For `at`, its only run.
+- `state` (String): `'active'`, `'suspended'`, `'sending'`, or `'ended'`. A delayed notification is `'sending'` once it has started to go out.
+- `suspendedReason` (String | null): Why it's [suspended](#suspended-and-ended-schedules).
+- `endReason` (String | null), `endedAt` (Number | null): Why and when it [ended](#suspended-and-ended-schedules).
+- `nextRunAt` (Number | null): The next slot of the schedule, worked out from its definition and the clock. A late or retried run for an earlier slot can still happen first. After the last slot is due, it stays at that slot until the run for it is done. It's `null` while the schedule is suspended or sending, and once it has ended.
+- `createdAt` (Number): When it was created.
+- `off()` (Function): Removes the schedule. Same as [`puter.events.removeSchedule(scheduleId)`](/Events/removeSchedule/). Only on the object `onSchedule()` returns.
+
+The promise rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `invalid_request` | `options` isn't an object. |
+| `events_handler_required` | There's no `handlerName`. |
+| `events_handler_name_required` | `handler` was given without `handlerName`. |
+| `events_handler_free_variable` | The handler uses an outside variable. The message names it. |
+| `events_handler_invalid` | `handler` isn't a function, a source string, or `{ file }`. |
+| `events_handler_hash_unavailable` | This environment has no `crypto.subtle`, so `handler` can't be hashed. Pass `handlerName` alone. |
+| `events_context_invalid` | `context` isn't JSON-serializable. |
+| `events_context_too_large` | The serialized `context` is over 4 KB. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `events_schedule_app_required` | The call wasn't made by an app, for example from Node.js with an account token. |
+| `events_durable_requires_account` | The call came from a scoped access token, such as the one in a [`getReadURL()`](/FS/getReadURL/) URL, or from a temporary (guest) account. |
+| `too_many_requests` | Over the create rate limit, which is shared with subscribing. |
+| `events_background_consent_required` | The user hasn't granted your app `events:background`. |
+| `events_schedule_invalid` | An option is malformed. For example, more than one of `every`, `cron` and `at`, an `every` that isn't a whole number of seconds, a time with no zone, an `at` less than 1 second or more than 366 days ahead, an unsupported cron expression, an unknown `tz`, or `tz` without `cron`. The message says which. |
+| `events_schedule_never_fires` | The schedule would never run. Its first run or its `startAt` is more than 366 days away, or it ends before its first run. |
+| `events_schedule_too_frequent` | It would run more often than your plan's [minimum interval](#how-often-a-schedule-can-run). |
+| `events_schedule_too_deep` | The call was made from a handler at the end of a [handler chain](/Events/onPersistent/#handlers-that-trigger-handlers), so the schedule could never run. |
+| `bad_request` | `handlerName` is empty or over 128 characters, or `handlerHash` isn't a SHA-256 hex digest. |
+| `events_handler_not_found` | Nothing is published under `handlerName`. The schedule isn't created. |
+| `events_handler_hash_mismatch` | The published handler isn't the source you passed as `handler`. |
+| `events_schedule_limit` | The user holds as many schedules as their plan allows, for your app or in total. |
+| `events_schedule_budget` | The schedule would make your app's schedules, or all of the user's schedules, run more times an hour than their plan allows. |
+| `events_schedule_busy` | Puter is checking too many cron expressions at once. Try again in a moment. |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+The handler is published once, by the app's developer, from a deploy script or the browser console. The schedule is created from the app, for each user.
+
+<strong class="example-title">Run a handler on weekday mornings in the user's time zone</strong>
+
+```js
+// Deploy script, run as the app's developer
+const app = await puter.apps.get('daily-digest');
+
+await puter.events.handlers.publish('sendDigest', async ({ event, ctx, user }) => {
+    const doneKey = `digest:${event.id}`;
+    if (await user.kv.get(doneKey)) return;
+
+    const notes = await user.kv.list('note:*', true);
+    await user.notifications.notify({
+        title: `Good morning, ${ctx.name}`,
+        text: `You have ${notes.length} notes.`,
+    });
+
+    await user.kv.set(doneKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}, { appUid: app.uid });
+```
+
+```js
+// In the app, signed in as the user
+await puter.perms.request('backgroundEvents');
+
+const schedule = await puter.events.onSchedule({
+    cron: '0 8 * * 1-5',
+    tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
+    handlerName: 'sendDigest',
+    context: { name: 'Ana' },
+});
+if (schedule.nextRunAt !== null) console.log(`next digest at ${new Date(schedule.nextRunAt).toString()}`);
+```
+
+<strong class="example-title">Run once, an hour from now</strong>
+
+```js
+const reminder = await puter.events.onSchedule({
+    at: Date.now() + 60 * 60 * 1000,
+    handlerName: 'remind',
+    context: { todoId: 'todo-42' },
+});
+
+// Changed your mind
+await reminder.off();
+```
+
+<strong class="example-title">Run every 6 hours, on the hour, 28 times</strong>
+
+```js
+await puter.events.onSchedule({
+    every: '6h',
+    startAt: '2026-01-01T00:00:00Z',
+    maxRuns: 28,
+    handlerName: 'syncFeeds',
+});
+```

src/docs/src/Events/removeSchedule.md

@@ -0,0 +1,72 @@
+---
+title: puter.events.removeSchedule()
+description: End a schedule, or cancel a notification stored to send later.
+platforms: [websites, apps, nodejs, workers]
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+Ends a schedule created with [`puter.events.onSchedule()`](/Events/onSchedule/). It doesn't run again, and it stays in [`listSchedules()`](/Events/listSchedules/) for 7 days with `endReason: 'removed'`. A run that already started isn't stopped.
+
+It also cancels a notification your app stored to send later with [`puter.notifications.notify()`](/Notifications/notify/#send-it-later), the same as [`puter.notifications.cancel()`](/Notifications/cancel/).
+
+The `off()` method on the schedule [`onSchedule()`](/Events/onSchedule/) returns does the same thing.
+
+On a website with nobody signed in, it asks the user to sign in first, as other Puter.js calls do. An app running on Puter is always signed in.
+
+## Syntax
+```js
+puter.events.removeSchedule(scheduleId)
+```
+
+## Parameters
+
+#### `scheduleId` (String) (required)
+The `scheduleId` from [`onSchedule()`](/Events/onSchedule/), [`notify()`](/Notifications/notify/#send-it-later) or [`listSchedules()`](/Events/listSchedules/).
+
+## Return value
+
+A `Promise` that resolves when the schedule has ended.
+
+An app can only remove schedules it created. An account session can remove any of the user's schedules, including ones left by apps they no longer use. A scoped access token can't remove any.
+
+An id this caller doesn't hold, or a schedule that already ended, is reported as not existing, so removing a schedule twice rejects the second time. With `events_schedule_sending`, `auth_canceled`, `too_many_requests` or `events_disabled`, nothing was removed. When the call fails any other way, such as a network error or `events_failed`, the removal may still have happened, so read the schedule again with [`getSchedule()`](/Events/getSchedule/) to find out.
+
+It rejects with `{ message, code }`:
+
+| `code` | Meaning |
+| --- | --- |
+| `events_schedule_not_found` | No such schedule, it isn't this caller's, or it already ended. Also for an empty `scheduleId`. |
+| `events_schedule_sending` | A notification stored to send later has started to go out, so it can't be cancelled. |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `too_many_requests` | Over the remove rate limit, which is shared with [`puter.events.unsubscribe()`](/Events/unsubscribe/). |
+| `events_disabled` | Schedules aren't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">Cancel a notification, then read it</strong>
+
+```html;events-remove-schedule
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            const later = await puter.notifications.notify({
+                title: 'Your trial ends tomorrow',
+                sendAt: Date.now() + 24 * 60 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            await puter.events.removeSchedule(later.scheduleId);
+            puter.print('cancelled<br>');
+
+            // The ended schedule can still be read for 7 days
+            const ended = await puter.events.getSchedule(later.scheduleId);
+            puter.print(`${ended.state} (${ended.endReason})<br>`);
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Events/subjects.md

@@ -176,6 +176,8 @@ A key-value change carries `key` instead of `uid` and `path`, and different ops:
 
 A notification carries the fields listed on [`fetch()`](/Events/fetch/#return-value).
 
+A [schedule](/Events/onSchedule/) has no subject you can subscribe to. Its runs carry `op: 'tick'` and the fields listed on [`onSchedule()`](/Events/onSchedule/#the-event).
+
 Events never say who made a change. On a shared folder, that would tell every subscriber who else has access.
 
 [`puter.kv.flush()`](/KV/flush/) delivers nothing, since the keys a flush can list aren't reliably the keys it removed.

src/docs/src/Events/unsubscribe.md

@@ -33,6 +33,7 @@ The promise rejects with `{ message, code }`:
 | `code` | Meaning |
 | --- | --- |
 | `subscription_does_not_exist` | No such subscription, or it isn't yours to end. |
+| `events_schedule_not_found` | The subscription belongs to a [schedule](/Events/onSchedule/). Remove it with [`removeSchedule()`](/Events/removeSchedule/) instead. |
 | `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
 | `too_many_requests` | Over the unsubscribe rate limit, which is separate from the subscribe one. |
 | `events_disabled` | Events aren't enabled on this server. |

src/docs/src/Events/update.md

@@ -36,6 +36,7 @@ The promise rejects with `{ message, code }`:
 | `code` | Meaning |
 | --- | --- |
 | `subscription_does_not_exist` | No such subscription, or it isn't yours to change. |
+| `events_schedule_immutable` | The subscription belongs to a [schedule](/Events/onSchedule/), and a schedule's `context` can't be changed. Remove the schedule and create a new one. |
 | `invalid_request` | `context` is missing. |
 | `events_context_too_large` | `context` is over 4 KB serialized. |
 | `events_context_invalid` | `context` can't be serialized as JSON. |

src/docs/src/Events/workers.md

@@ -6,7 +6,7 @@ platforms: [websites, apps, nodejs, workers]
 
 <div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
 
-An **events worker** runs an app's published [handlers](/Events/handlers/). It runs them for every event on a subscription with a `handlerName`, whether or not a client is connected, and for a [`queue`](/Events/onPersistent/#choosing-a-mode) delivery once no connected client takes it. Each app has at most one, created when it publishes its first handler, however many handlers it publishes.
+An **events worker** runs an app's published [handlers](/Events/handlers/). It runs them for every event on a subscription with a `handlerName`, whether or not a client is connected, for a [`queue`](/Events/onPersistent/#choosing-a-mode) delivery once no connected client takes it, and for every run of a [schedule](/Events/onSchedule/). Each app has at most one, created when it publishes its first handler, however many handlers it publishes.
 
 A hosted Puter server may bill each app's events worker as a monthly cost, even if nothing ever delivers to it. Use this API to see which apps have one and remove the ones you don't need.
 
@@ -43,11 +43,11 @@ Resolves to `{ items, cursor, deployable }`:
 puter.events.workers.destroy(appUid)
 ```
 
-Removes every handler the app has published, the same as calling [`puter.events.handlers.remove()`](/Events/handlers/) on each. Their subscriptions are suspended with `handler_not_found`, not deleted, and publishing the handlers again resumes them.
+Removes every handler the app has published, the same as calling [`puter.events.handlers.remove()`](/Events/handlers/) on each. Their subscriptions and schedules are suspended with `handler_not_found`, not deleted, and publishing the handlers again resumes them.
 
 It also ends the worker session background deliveries ran under, for every user. The next delivery after a new publish starts a new one.
 
-Resolves to `{ appUid, removed, suspended }`, the number of handlers deleted and the number of subscriptions suspended. An app with nothing published rejects with `events_handler_not_found`.
+Resolves to `{ appUid, removed, suspended }`, the number of handlers deleted and the number of subscriptions and schedules suspended. An app with nothing published rejects with `events_handler_not_found`.
 
 ## Errors
 

src/docs/src/Notifications.md

@@ -29,6 +29,8 @@ A notification has a `title`, and optionally `text` and an `icon`. A `target` pa
 
 [`notify()`](/Notifications/notify/) lists every field. [Rate Limits and Quotas](/rate-limits-and-quotas/#notifications) lists the size limits and how many notifications and pushes your app can send each user.
 
+To send a notification at a set time instead of now, pass `sendAt` to [`notify()`](/Notifications/notify/#send-it-later). It goes out then, even when your app is closed.
+
 ## Who you can notify
 
 A notification goes to the user your app runs for. To notify another user, such as a chat member, share data with them in a [key-value share](/KV/share/) made with `notify: true`, and send through the share's handle. Each side of the share can notify the other.
@@ -134,6 +136,7 @@ Your app's notifications are events on the `notif:app-user` [subject](/Events/su
 ## Functions
 
 - **[`puter.notifications.notify()`](/Notifications/notify/)** - Send a notification to the user, or to the other side of a key-value share
+- **[`puter.notifications.cancel()`](/Notifications/cancel/)** - Cancel a notification stored to send later
 - **[`puter.notifications.onNotification()`](/Notifications/onNotification/)** - Receive your app's notifications, including the ones sent while it was closed
 - **[`puter.notifications.onPush()`](/Notifications/onPush/)** - Say what a push shows for one channel, and handle its notifications while the page is open
 - **[`puter.notifications.on('click')`](/Notifications/on/)** - React when the user clicks one of your notifications
@@ -152,6 +155,7 @@ For Puter's own apps, which manage where pushes go:
 ## Examples
 
 - [Send a Notification](/playground/notifications-notify/)
+- [Send a Notification Later](/playground/notifications-send-later/)
 - [Push notifications for a website](/Notifications/push/#a-minimal-page)
 
 ## Recipes

src/docs/src/Notifications/cancel.md

@@ -0,0 +1,64 @@
+---
+title: puter.notifications.cancel()
+description: Cancel a notification your app stored to send later, before it goes out.
+platforms: [websites, apps]
+---
+
+Cancels a notification your app stored to send later with [`notify()`](/Notifications/notify/#send-it-later) and `sendAt`. It isn't sent, and it stops counting toward the [limit on what your app has waiting](/rate-limits-and-quotas/#schedules) for the user.
+
+It does the same thing as [`puter.events.removeSchedule()`](/Events/removeSchedule/), and shares its rate limit with it. The cancelled notification stays in [`puter.events.listSchedules()`](/Events/listSchedules/) for 7 days, with `state: 'ended'` and `endReason: 'removed'`.
+
+A notification that has started to go out can't be cancelled. Once it's sent, it's in the user's notifications like any other.
+
+## Syntax
+```js
+puter.notifications.cancel(scheduleId)
+```
+
+## Parameters
+
+#### `scheduleId` (String) (required)
+The `scheduleId` that [`notify()`](/Notifications/notify/#send-it-later) resolved with.
+
+## Return value
+
+A `Promise` that resolves when the notification is cancelled.
+
+It rejects with an error carrying `message` and `code`:
+
+| `code` | Meaning |
+| --- | --- |
+| `events_schedule_not_found` | No such notification, it isn't your app's, or it was already sent or cancelled. Also for an empty `scheduleId`. |
+| `events_schedule_sending` | It has started to go out, so it can't be cancelled. |
+| `too_many_requests` | Over the cancel rate limit, which is shared with [`puter.events.removeSchedule()`](/Events/removeSchedule/) and [`puter.events.unsubscribe()`](/Events/unsubscribe/). |
+| `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
+| `events_disabled` | Sending later isn't available on this server, as on self-hosted servers. |
+| `events_failed` | The server sent a response the SDK couldn't read. |
+
+## Examples
+
+<strong class="example-title">Store a reminder, then cancel it</strong>
+
+```html;notifications-send-later
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            // (1) Send it in 30 minutes instead of now
+            const later = await puter.notifications.notify({
+                title: 'Your timer is done',
+                target: '/timers',
+                sendAt: Date.now() + 30 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+            puter.print(`stored ${later.scheduleId}, goes out ${new Date(later.sendAt).toLocaleTimeString()}<br>`);
+
+            // (2) The user stopped the timer, so it shouldn't go out
+            await puter.notifications.cancel(later.scheduleId);
+            puter.print('cancelled<br>');
+        })();
+    </script>
+</body>
+</html>
+```

src/docs/src/Notifications/notify.md

@@ -23,13 +23,14 @@ puter.notifications.notify(notification)
 - `text` (String): Up to 240 characters. Line breaks are kept.
 - `icon` (String): An `https` URL to an image, up to 2,048 characters. Without it, your app's icon is used. A push that Puter shows always uses your app's icon.
 - `target` (String): A path inside your app, starting with `/`, such as `/chats/42`. Up to 512 characters. A click hands it back to your app through [`on('click')`](/Notifications/on/). On a website, a tap on a push opens this path when no page of your site is open.
-- `data` (any): A JSON value of up to 1 KB, handed back on click. Push services can read it, so put IDs in it, not personal content.
+- `data` (any): A JSON value of up to 1 KB, handed back on click. Push services can read it, so put ids in it, not personal content.
 - `group` (String): A later notification with the same `group` replaces this one, as [Groups](#groups) describes. Up to 128 letters, digits and `. _ : @ -`.
 - `channel` (String): What kind of notification it is, such as `messages` or `reminders`. The [`onPush()`](/Notifications/onPush/) handler registered with this key decides what its push shows. Up to 32 lowercase letters, digits and `-`.
-- `sensitive` (Boolean): Pushes, and the notification a page of your app shows itself, name only your app, such as "New notification from Chat" for an app named Chat, instead of showing the title and text. On a website, a sensitive push carries no `channel`, `target` or `data`. So no [`onPush()`](/Notifications/onPush/) handler runs for it, a tap opens your service worker's scope (`/` by default), and [`on('click')`](/Notifications/on/) gets `null` for `target` and `data`. [`onNotification()`](/Notifications/onNotification/) still gets the whole notification.
+- `sensitive` (Boolean): Pushes, and the notification a page of your app shows itself, name only your app, such as "New notification from Chat" for an app named Chat, instead of showing the title and text. On a website, a sensitive push carries no `channel`, `target` or `data`. So no [`onPush()`](/Notifications/onPush/) handler runs for it, a tap opens your service worker's scope (`/` by default), and [`on('click')`](/Notifications/on/) gets `null` for `target` and `data`. A listener added with [`onNotification()`](/Notifications/onNotification/) still gets the whole notification.
 - `priority` (String): Only `'normal'` is available to apps.
 - `ttl` (Number): How many seconds a push may wait for a device that's offline, from 1 to 86400. Defaults to 3600. It doesn't change how long the notification is kept.
 - `to` (Object): Send it to the other side of a key-value share instead, as `{ handle }` with the share's handle. See [Notify the other side of a share](#notify-the-other-side-of-a-share).
+- `sendAt` (Number | String | Date): Send it at this time instead of now, as a `Date`, an ISO-8601 string with a zone, a date alone (midnight UTC), or whole milliseconds since the epoch. Rounded up to the next whole second, and at most 366 days ahead. A time within 5 seconds, or in the past, sends it now. See [Send it later](#send-it-later).
 
 ## Return value
 
@@ -37,6 +38,8 @@ A `Promise` that resolves to the notification's `uid` (String). When the notific
 
 With `to`, the `Promise` resolves with no value, so the call doesn't reveal anything about the recipient's notifications.
 
+With a `sendAt` more than 5 seconds ahead, the `Promise` resolves to `{ scheduleId, sendAt }` instead, with or without `to`. To cancel it before it goes out, pass `scheduleId` to [`cancel()`](/Notifications/cancel/). The `sendAt` it resolves with is when it goes, in milliseconds since the epoch. With a `sendAt` within 5 seconds, it resolves as it would without one.
+
 It rejects with an error carrying `message` and `code`:
 
 | `code` | Meaning |
@@ -44,11 +47,15 @@ It rejects with an error carrying `message` and `code`:
 | `invalid_argument` | `notification` isn't an object, or its `title` is missing or blank. |
 | `bad_request` | A field is over its limit, has the wrong type, or isn't one of the fields above. The message names the field. With `to`, it must be an object with only a string `handle`. |
 | `forbidden` | The call wasn't made by an app, for example from Node.js with an account token. |
-| `too_many_requests` | Your app sent this user too many notifications. With `to`, the error is the same whether the recipient's limit or the [share limit](#notify-the-other-side-of-a-share) refused the call. |
+| `too_many_requests` | Your app sent this user too many notifications. With `to`, the error is the same whether the recipient's limit or the [share limit](#notify-the-other-side-of-a-share) refused the call. With a `sendAt` more than 5 seconds ahead, the user stored too many notifications or created too many [schedules](/Events/onSchedule/) in the last minute. See [Rate Limits and Quotas](/rate-limits-and-quotas/#schedules). |
 | `subject_does_not_exist` | With `to`, the handle doesn't name a live share of this app's data that you're on either side of. |
 | `kv_share_notify_not_enabled` | With `to`, the share wasn't made with `notify: true`. |
 | `recipient_not_accepting_shares` | With `to`, the recipient blocked you or doesn't accept shares. |
 | `service_unavailable` | With `to`, Puter couldn't check the share. Try again later. |
+| `events_schedule_invalid` | `sendAt` isn't a valid time, has no zone, or is more than 366 days ahead. |
+| `events_durable_requires_account` | With `sendAt`, the call came from a scoped access token or a temporary (guest) account, even when `sendAt` is due now. |
+| `events_schedule_limit` | With `sendAt`, your app, or all apps together, have as many notifications and [`at` schedules](/Events/onSchedule/) waiting for this user as the plan allows. |
+| `events_disabled` | With `sendAt`, sending later isn't available on this server, as on self-hosted servers, even when `sendAt` is due now. |
 | `auth_canceled` | Nobody was signed in, and the user closed the sign-in without finishing it. |
 
 ## Notify the other side of a share
@@ -59,6 +66,41 @@ The notification counts against the recipient's limits from your app, the same a
 
 The notification carries your app's name, not the sender's. Put the sender in `title` or `text` if the recipient should see it.
 
+## Send it later
+
+<div class="info">Sending later with <code>sendAt</code> is available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+With `sendAt`, the notification is stored and sent at that time, even when your app is closed and no page of it is open:
+
+```js
+const later = await puter.notifications.notify({
+    title: 'Your free trial ends tomorrow',
+    target: '/billing',
+    sendAt: '2026-11-14T09:00:00-05:00',
+});
+
+// The user upgraded, so it shouldn't go out. A sendAt that's due now was
+// sent at once and resolved with its uid instead.
+if (typeof later === 'object' && later) await puter.notifications.cancel(later.scheduleId);
+```
+
+A stored notification is a schedule of `kind: 'deferred'`. You can read it with [`puter.events.getSchedule()`](/Events/getSchedule/) and [`puter.events.listSchedules()`](/Events/listSchedules/), which show `nextRunAt` set to its `sendAt` until it starts to go out. It needs no handler and no `events:background` permission.
+
+When it's due:
+
+- It usually goes out within 15 seconds of `sendAt`, as if [`notify()`](/Notifications/notify/) were called then. It reaches every open page of your app live, and goes out as a push when no page answers, as described in [Push notifications for your app](/Notifications/push/#when-a-notification-is-pushed). A `group` replaces whatever notification is open in that group at that time.
+- The notification limits are counted then, against the recipient. With `to`, the share is checked when you call [`notify()`](/Notifications/notify/), and again when it goes out, which is when it counts toward the limit for that sender, recipient and app. A push is billed to the user who receives it, as any push is.
+- If it can't go out because of a rate limit or a temporary error, it's retried, up to 5 attempts in all. After that it's dropped. A share that stopped, a recipient who blocked you, or an app or account that was deleted drops it right away.
+- While the sending account is suspended, it waits, and it's tried again every 15 minutes.
+
+It's written once, even when sending is retried. A retry doesn't write a second notification, and doesn't push it or bill it a second time. An open page of your app can receive it twice, with the same `uid`. A retry delivers it to open pages again only within 24 hours of the first attempt. After that, the retry finishes without delivering it.
+
+Once it has gone out, or been dropped, its schedule has `state: 'ended'` and an `endReason` of `delivered` or `dropped`, and it stays in [`puter.events.listSchedules()`](/Events/listSchedules/) for 7 days. While it's going out it's `sending`, and [`cancel()`](/Notifications/cancel/) rejects with `events_schedule_sending`.
+
+When the user uninstalls your app, the notifications it has waiting for them are cancelled, and one that's due after the uninstall is dropped. Deleting the app, or the account that owns it, cancels all of them. This holds even on a server where sending later is turned off.
+
+The number of notifications and [`at` schedules](/Events/onSchedule/) your app can have waiting for each user is limited by the user's plan. See [Rate Limits and Quotas](/rate-limits-and-quotas/#schedules).
+
 ## Groups
 
 A notification with a `group` replaces the newest notification your app sent the same user with that `group`, as long as that one hasn't been shown or dismissed. The replacement keeps that notification's `uid` and takes the new content and time. Its push replaces the earlier one on the device and alerts the user again. A listener added with [`onNotification()`](/Notifications/onNotification/) gets the replacement too, with the same `uid` and `replaced: true`.

src/docs/src/Notifications/push.md

@@ -131,7 +131,9 @@ await puter.notifications.notify({
 The page you send from is usually open and focused. With an [`onPush()`](/Notifications/onPush/) or [`onNotification()`](/Notifications/onNotification/) listener, it gets the notification live and nothing is pushed. To reach the user while your app is closed, send from somewhere that keeps running:
 
 - **A [Puter Worker](/Workers/)**, with [`user.puter.notifications.notify()`](/Notifications/notify/) in a route your page calls through [`puter.workers.exec()`](/Workers/exec/). It notifies the user who called the worker, from the app that called it.
-- **An [events handler](/Events/handlers/)**, with [`user.notifications.notify()`](/Notifications/notify/). It notifies the user who holds the subscription, and it runs while your app is closed. Running a handler in the background needs the user's `events:background` permission, as [`onPersistent()`](/Events/onPersistent/) describes.
+- **An [events handler](/Events/handlers/)**, with [`user.notifications.notify()`](/Notifications/notify/). It notifies the user who holds the subscription or [schedule](/Events/onSchedule/), and it runs while your app is closed. Running a handler in the background needs the user's `events:background` permission, as [`onPersistent()`](/Events/onPersistent/) describes.
+
+To send a notification at a set time, pass `sendAt` to [`notify()`](/Notifications/notify/#send-it-later). Puter stores it and sends it then, even with your app closed, and [`cancel()`](/Notifications/cancel/) cancels it until it goes out. It counts toward the push limits and is billed when it goes out, not when you store it. When sending it has to be retried, it still goes out as one push, billed once, though an open page of your app can receive it twice, with the same `uid`. To send something different each time, such as a daily summary, run a handler on a timer with [`puter.events.onSchedule()`](/Events/onSchedule/) and call [`user.notifications.notify()`](/Notifications/notify/) from it.
 
 An app that notifies from an events handler needs the user to allow both background events and pushes. Ask for both with [`puter.perms.request()`](/Perms/request/#batching), so the user answers one prompt:
 
@@ -306,7 +308,7 @@ The browser can be allowed to show notifications while the operating system hide
 ## Good to know
 
 - Say who did what. "Ana shared Q3-report.pdf" reads well as a push. A push that only says there's something new can be hidden as spam on Android.
-- The `data` field is handed back on click, and push services can read it, so put IDs in it and fetch the rest. Set `sensitive: true` when the title and text shouldn't leave Puter.
+- The `data` field is handed back on click, and push services can read it, so put ids in it and fetch the rest. Set `sensitive: true` when the title and text shouldn't leave Puter.
 - A push carries about 4 KB, with `data` at most 1 KB. A push that doesn't fit has its text shortened.
 - A tap opens your domain. The `url` a handler returns has to be on your origin, and a tap on one from another origin opens your worker's folder instead.
 - Pushes from your app are billed to the user who receives them. See [What pushes cost](/rate-limits-and-quotas/#what-pushes-cost). Puter's own notifications are free.

src/docs/src/examples.js

@@ -676,6 +676,24 @@ const examples = [
                 slug: 'events-unsubscribe',
                 source: '/playground/examples/events-unsubscribe.html',
             },
+            {
+                title: 'List Schedules',
+                description: 'List the schedules and stored notifications an app holds for a user with the Puter.js Events API. Run and experiment with this example in the playground.',
+                slug: 'events-list-schedules',
+                source: '/playground/examples/events-list-schedules.html',
+            },
+            {
+                title: 'Read a Schedule',
+                description: 'Read one schedule and see when it runs next with the Puter.js Events API. Try it in the playground.',
+                slug: 'events-get-schedule',
+                source: '/playground/examples/events-get-schedule.html',
+            },
+            {
+                title: 'Remove a Schedule',
+                description: 'End a schedule with the Puter.js Events API, then read the ended schedule. Run it in the playground.',
+                slug: 'events-remove-schedule',
+                source: '/playground/examples/events-remove-schedule.html',
+            },
             {
                 title: 'Catch Up on Notifications',
                 description: 'Page through notifications missed while your app was closed with the Puter.js Events API. Run it in the playground.',
@@ -699,6 +717,12 @@ const examples = [
                 slug: 'notifications-notify',
                 source: '/playground/examples/notifications-notify.html',
             },
+            {
+                title: 'Send a Notification Later',
+                description: 'Store a notification to send at a set time, then cancel it, with the Puter.js Notifications API. Run it in the playground.',
+                slug: 'notifications-send-later',
+                source: '/playground/examples/notifications-send-later.html',
+            },
         ],
     },
     {

src/docs/src/playground/examples/events-get-schedule.html

@@ -0,0 +1,23 @@
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            const later = await puter.notifications.notify({
+                title: 'Stand-up in 10 minutes',
+                sendAt: Date.now() + 10 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            const schedule = await puter.events.getSchedule(later.scheduleId);
+            const due = schedule.nextRunAt ? new Date(schedule.nextRunAt).toLocaleTimeString() : 'nothing due';
+            puter.print(`${schedule.state}, due ${due}<br>`);
+
+            // Ended schedules can still be read
+            await puter.notifications.cancel(later.scheduleId);
+            const ended = await puter.events.getSchedule(later.scheduleId);
+            puter.print(`${ended.state} (${ended.endReason})<br>`);
+        })();
+    </script>
+</body>
+</html>

src/docs/src/playground/examples/events-list-schedules.html

@@ -0,0 +1,29 @@
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            // (1) Something to list: a notification to send in an hour
+            const later = await puter.notifications.notify({
+                title: 'Time for a break',
+                sendAt: Date.now() + 60 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            // (2) Page through everything this app holds for the user
+            let cursor = null;
+            do {
+                const page = await puter.events.listSchedules({ cursor, limit: 50 });
+                for (const schedule of page.items) {
+                    const next = schedule.nextRunAt ? new Date(schedule.nextRunAt).toLocaleString() : 'nothing due';
+                    puter.print(`${schedule.kind} ${schedule.state}, next: ${next}<br>`);
+                }
+                cursor = page.cursor ?? null;
+            } while (cursor);
+
+            // (3) Cleanup
+            await puter.notifications.cancel(later.scheduleId);
+        })();
+    </script>
+</body>
+</html>

src/docs/src/playground/examples/events-remove-schedule.html

@@ -0,0 +1,21 @@
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            const later = await puter.notifications.notify({
+                title: 'Your trial ends tomorrow',
+                sendAt: Date.now() + 24 * 60 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+
+            await puter.events.removeSchedule(later.scheduleId);
+            puter.print('cancelled<br>');
+
+            // The ended schedule can still be read for 7 days
+            const ended = await puter.events.getSchedule(later.scheduleId);
+            puter.print(`${ended.state} (${ended.endReason})<br>`);
+        })();
+    </script>
+</body>
+</html>

src/docs/src/playground/examples/notifications-send-later.html

@@ -0,0 +1,21 @@
+<html>
+<body>
+    <script src="https://js.puter.com/v2/"></script>
+    <script>
+        (async () => {
+            // (1) Send it in 30 minutes instead of now
+            const later = await puter.notifications.notify({
+                title: 'Your timer is done',
+                target: '/timers',
+                sendAt: Date.now() + 30 * 60 * 1000,
+            });
+            if (typeof later !== 'object' || !later) return;    // due now, so it was sent at once
+            puter.print(`stored ${later.scheduleId}, goes out ${new Date(later.sendAt).toLocaleTimeString()}<br>`);
+
+            // (2) The user stopped the timer, so it shouldn't go out
+            await puter.notifications.cancel(later.scheduleId);
+            puter.print('cancelled<br>');
+        })();
+    </script>
+</body>
+</html>

src/docs/src/rate-limits-and-quotas.md

@@ -589,7 +589,7 @@ Calls over any of these fail with `too_many_requests`. A share over the `kv.shar
 
 A [`puter.kv.share()`](/KV/share/) call names 1 to 50 recipients, and each one counts toward the named-recipient limit. A recipient who doesn't already have the share also counts toward the new-recipient limit, including a username that doesn't exist and a user the share is refused for. The call counts once toward the mint limit when it makes a share, changes one, or has any recipient who counts as new. A call where every recipient already has the share at the same access level and label counts only toward the named-recipient limit. When the recipients don't all fit in a limit, the whole call fails with `too_many_requests`, and the part that did fit still counts.
 
-Unsubscribing counts toward its own `unsubscribe` limit, not the `subscribe` one. Changing a subscription's context with [`puter.events.update()`](/Events/update/) counts toward the `subscribe` limit.
+Unsubscribing counts toward its own `unsubscribe` limit, not the `subscribe` one. Changing a subscription's context with [`puter.events.update()`](/Events/update/) counts toward the `subscribe` limit. Creating a [schedule](#schedules), or storing a notification with [`notify()`](/Notifications/notify/#send-it-later) and a `sendAt` more than 5 seconds ahead, counts toward the `subscribe` limit. Removing one counts toward `unsubscribe`, and listing or reading one toward the subscription listings.
 
 #### Delivery
 
@@ -660,6 +660,69 @@ When a kept backlog expires, it's replaced by one gap marker (`suspended_backlog
 
 A pattern over its limits fails with `invalid_subject_pattern`, and a context over 4 KB with `events_context_too_large`.
 
+#### Schedules
+
+These limits apply to [`onSchedule()`](/Events/onSchedule/) schedules, and to notifications stored with [`notify()`](/Notifications/notify/#send-it-later) and `sendAt`. These are puter.com's limits.
+
+| Limit                                                        | Scope              | Paid   | Free  | Anonymous |
+| ------------------------------------------------------------ | ------------------ | ------ | ----- | --------- |
+| Recurring schedules (`every` and `cron`)                     | Per user, per app  | 100    | 10    | 0         |
+| Recurring schedules                                          | Per user, all apps | 1,000  | 50    | 0         |
+| `at` schedules and stored notifications waiting to go out    | Per user, per app  | 1,000  | 100   | 0         |
+| `at` schedules and stored notifications waiting to go out    | Per user, all apps | 5,000  | 300   | 0         |
+| Runs per hour of all recurring schedules added up            | Per user, per app  | 1,800  | 40    | 0         |
+| Runs per hour of all recurring schedules added up            | Per user, all apps | 18,000 | 120   | 0         |
+
+A recurring schedule can run at most every 1 minute on a paid plan and every 15 minutes on a free one. For a `cron` schedule, that's the smallest gap between two of its runs.
+
+- A create over a count fails with `events_schedule_limit`, one over the runs per hour with `events_schedule_budget`, and one under the minimum interval with `events_schedule_too_frequent`. Anonymous accounts get `events_durable_requires_account`.
+- A recurring schedule's runs per hour are worked out when it's created. An `every` schedule counts 3,600 divided by its interval in seconds, rounded up to the next 0.001, so one that runs every minute counts 60, one that runs every 15 minutes counts 4, and one that runs every 7 minutes counts 8.572. A `cron` schedule counts the most runs it makes in any one hour, over its next 10,000 runs or 400 days. So a paid plan fits 30 schedules that run every minute in one app, and a free plan fits 10 that run every 15 minutes.
+- One-time `at` schedules and stored notifications count only toward the waiting limits, until they run or are cancelled.
+- A suspended schedule still counts. A schedule stops counting as soon as it's removed or its last run finishes.
+- Schedules don't count toward the persistent subscription limits.
+- These limits are checked when a schedule is created, against the plan the user has then. Schedules made before a change to a plan with lower limits keep running. New ones are held to the current plan.
+
+| Limit                                              | Value                                   | Scope              |
+| -------------------------------------------------- | --------------------------------------- | ------------------ |
+| Schedule runs                                      | 200 per minute                          | Per user, per app  |
+| Schedule runs at once                              | 8 per server                            | Per user, all apps |
+| Time from a run's slot until it starts             | Usually within 60 seconds               | Per run            |
+| Spread after the slot, recurring                   | Under 30 seconds, or under a tenth of the smallest gap if that's less | Per schedule |
+| Spread after the slot, `at` and stored notifications | Under 10 seconds                      | Per schedule       |
+| Run timeout                                        | 30 seconds                              | Per run            |
+| Retry delay                                        | 2 seconds, doubling, up to 5 minutes    | Per run            |
+| Failures in a row before suspension                | 5                                       | Per schedule       |
+| Wait while the user's account is suspended         | 15 minutes, then tried again            | Per run            |
+| `missed` count                                     | Up to 1,000                             | Per run            |
+| `maxRuns`                                          | 1 to 100,000                            | Per schedule       |
+| `every`, and how far ahead `at`, `startAt`, the first run and `sendAt` can be | 366 days     | Per schedule       |
+| Cron expression                                    | 128 characters                          | Per schedule       |
+| Runs checked when a `cron` schedule is created     | 10,000 runs or 400 days, whichever ends first | Per schedule |
+| `cron` schedules checked at once                   | 2, with 8 more waiting                  | Per server         |
+| `context`                                          | 4 KB                                    | Per schedule       |
+| Events worker deploys started by schedule runs     | 8 at once                               | Per region         |
+| Ended schedules kept in `listSchedules()`          | 7 days                                  | Per schedule       |
+| A stored notification sent within 5 seconds        | Sent right away instead                 | Per notification   |
+| Attempts to send a stored notification             | 5, then it's dropped                    | Per notification   |
+| A stored notification over its recipient's limits  | Dropped 6 hours after `sendAt`          | Per notification   |
+| A stored notification, as JSON                     | 16 KB                                   | Per notification   |
+
+- Over the schedule runs per minute, runs wait for the next minute. They don't fail and don't count as failures. This limit is separate from the 60 handler runs a minute for subscriptions, so neither one uses up the other.
+- Over the runs at once, a run waits about 3 seconds and tries again. It doesn't fail, doesn't count as a failure, and keeps its `event.id` and `missed` count.
+- A run that finds 8 deploys already started waits 5 seconds and tries again.
+- A stored notification over its recipient's [notification limits](#notifications) waits at least a minute and tries again, without using one of its 5 attempts.
+- A `cron` create that finds 2 checks running and 8 waiting fails with `events_schedule_busy`. Try again in a moment.
+- A run that Puter couldn't get to its handler is retried until the schedule's next run is due, and then it's dropped. An `at` schedule's run, or a schedule's last run, is retried for 24 hours. A dropped run isn't billed and doesn't count as a failure.
+- A stored notification counts toward the [notification limits](#notifications) of its recipient when it goes out, not when it's stored.
+
+| Call                                                | Limit                                | Scope              |
+| --------------------------------------------------- | ------------------------------------ | ------------------ |
+| `onSchedule()`, and `notify()` with a later `sendAt` | 60/min, shared with `subscribe`     | Per user, all apps |
+| `removeSchedule()` and `notifications.cancel()`     | 600/min, shared with `unsubscribe`   | Per user, all apps |
+| `listSchedules()` and `getSchedule()`               | 120/min, shared with the subscription listings | Per user, per app |
+
+A [`listSchedules()`](/Events/listSchedules/) page holds 50 schedules by default, and up to 200. Calls over any of these limits fail with `too_many_requests`. A create refused for its options, its handler or a missing permission still counts toward the [`onSchedule()`](/Events/onSchedule/) limit, and removing a schedule that doesn't exist still counts toward the [`removeSchedule()`](/Events/removeSchedule/) limit.
+
 #### What events cost
 
 Deliveries are billed to the account holding the subscription:
@@ -671,9 +734,11 @@ Deliveries are billed to the account holding the subscription:
 
 µ¢ is a microcent (a millionth of a cent). Deliveries for `queue` subscriptions bill on the `single` line, and every other delivery, session subscriptions included, bills at the broadcast rate. Handler runs bill separately as worker usage.
 
-Nothing is billed for idle subscriptions, events a filter excluded, writes merged by coalescing, deliveries stopped by a permission check, deliveries with no client connected to receive them and no handler run, `queue` deliveries still waiting to reach a client or a handler, retries (a `queue` event bills once however many attempts it takes), and gap markers.
+Each [schedule](/Events/onSchedule/) run bills once on the `single` line, to the user holding the schedule, however many attempts it takes. That includes a run whose handler threw. A schedule that runs every minute costs about $0.04 a month on top of its handler's worker usage. A [stored notification](/Notifications/notify/#send-it-later) bills nothing until it goes out, and then only its push, as [What pushes cost](#what-pushes-cost) describes.
 
-When the holder's balance runs out, deliveries stop and persistent subscriptions are suspended with `no_credit`, and the holder is notified. The backlog is kept for 1 hour. Topping up resumes them within a few minutes.
+Nothing is billed for idle subscriptions and schedules, skipped schedule runs (counted in `missed`), schedule runs stopped by the handler chain limit, events a filter excluded, writes merged by coalescing, deliveries stopped by a permission check, deliveries with no client connected to receive them and no handler run, `queue` deliveries still waiting to reach a client or a handler, retries (a `queue` event bills once however many attempts it takes), and gap markers.
+
+When the holder's balance runs out, deliveries stop and persistent subscriptions and schedules are suspended with `no_credit`, and the holder is notified. The backlog is kept for 1 hour. Topping up resumes them within a few minutes.
 
 ### Networking
 

src/docs/src/recipes/change-data-at-a-set-time.md

@@ -0,0 +1,414 @@
+---
+title: Change Data at a Set Time
+description: "Learn how to change a user's data at a time you pick with Puter.js, and cancel or move the change later, so it happens on time even while your app is closed."
+tags: [events, notifications, kv]
+order: 69
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+<div class="info">Schedules are available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+Some changes are due at a set time instead of right away. For example, a
+snoozed task comes back to the list, a poll closes, or a shared link stops
+working. On other platforms you'd store the time in a database and run a job
+that checks it every minute.
+
+In Puter, you publish a function once, and create a one-time schedule that
+runs it at that time with [`puter.events.onSchedule()`](/Events/onSchedule/)
+and `at`. Puter runs it in your app's [events worker](/Events/workers/), a
+serverless function that belongs to your app, even while your app is closed.
+
+When all that's due is a notification,
+[Remind Users Later](/recipes/remind-users-later/) sends one at a set time
+with no handler or permission. To delete a key at a set time, give it an
+expiry time, as [Store Temporary Data](/recipes/store-temporary-data/) shows.
+
+The examples add snoozing to a task list app. The user hides a task until a
+time they pick. At that time the task comes back to the list, and the user
+gets a notification about it.
+
+## Publish the Handler
+
+The function a schedule runs is called a handler. Publish it with
+[`puter.events.handlers.publish()`](/Events/handlers/). This is a developer
+step, like a deploy, and it only works for the account that owns the app. Run
+it from a deploy script or your browser console:
+
+```js
+const app = await puter.apps.get('task-list');    // your app's name
+
+async function unsnooze ({ ctx, user }) {
+    const key = `task:${ctx.taskId}`;
+    const task = await user.kv.get(key);
+    if (!task) return;    // deleted while it was snoozed
+
+    await user.notifications.notify({
+        title: 'Back on your list',
+        text: task.title,
+        target: `/tasks/${ctx.taskId}`,
+    });
+    await user.kv.update(key, { snoozedUntil: null });
+}
+
+await puter.events.handlers.publish('unsnooze', unsnooze, { appUid: app.uid });
+```
+
+The handler gets `ctx`, the `context` the schedule was created with, and
+`user`, a `puter` object signed in as the user the schedule belongs to. It
+reads the task from the user's [key-value store](/KV/), sends a notification
+with [`user.notifications.notify()`](/Notifications/notify/), and marks the
+task as no longer snoozed with [`user.kv.update()`](/KV/update/).
+[Run Code on a Schedule](/recipes/run-code-on-a-schedule/#publish-the-handler)
+explains what else a handler can use.
+
+## Ask for Permission
+
+A schedule runs code while the user isn't there, so it needs the user's
+`events:background` permission. Ask from your app with
+[`puter.perms.request()`](/Perms/request/):
+
+```js
+const allowed = await puter.perms.request('backgroundEvents');
+```
+
+It resolves to `true` or `false`, and shows no prompt when the permission is
+already granted. Without it, [`onSchedule()`](/Events/onSchedule/) fails with
+`events_background_consent_required`. Schedules also need a Puter account,
+so for a guest account [`onSchedule()`](/Events/onSchedule/) fails with
+`events_durable_requires_account`.
+
+If the user revokes the permission later, every schedule your app holds for
+them ends, and their snoozed tasks stay snoozed. When the app opens, it can
+bring back any task whose `snoozedUntil` has passed.
+
+## Snooze a Task
+
+To snooze a task, create a schedule in your app with
+[`puter.events.onSchedule()`](/Events/onSchedule/). The `at` option is the time
+it runs, once:
+
+```js
+async function snooze (taskId, until) {
+    const schedule = await puter.events.onSchedule({
+        at: until,
+        handlerName: 'unsnooze',
+        context: { taskId },
+    });
+    await puter.kv.update(`task:${taskId}`, {
+        snoozedUntil: schedule.startAt,
+        snoozeId: schedule.scheduleId,
+    });
+}
+
+if (allowed) await snooze('42', new Date(Date.now() + 3 * 60 * 60 * 1000));    // three hours from now
+```
+
+The `at` option takes a `Date`, an ISO-8601 string with a time zone, or
+milliseconds since the epoch, from 1 second to 366 days ahead. Puter rounds it
+up to the next whole second, and `schedule.startAt` is the time the schedule
+runs, in milliseconds since the epoch. A run usually starts within a minute of
+it.
+
+The handler gets `context` as `ctx`. The `context` is read once, when the
+schedule is created, can't be changed later, and can be at most 4 KB. So it
+holds only the task's id, and the handler reads the task when it runs. A task
+renamed while it's snoozed comes back with its new name.
+
+The task keeps the schedule's id in `snoozeId`. The next sections use it to
+skip repeated runs and to cancel or move the snooze.
+
+A user can have a limited number of one-time schedules waiting for your app.
+Past it, [`onSchedule()`](/Events/onSchedule/) rejects with
+`events_schedule_limit`. See
+[Rate Limits and Quotas](/rate-limits-and-quotas/#schedules).
+
+## Skip a Run Already Done
+
+Each run happens at least once. A run that times out, or fails with a network
+error, is retried, and a retry can repeat work the first attempt already did.
+Every attempt at a run has the same `event.id`, `<scheduleId>:<scheduledFor>`.
+A one-time schedule has a single run, so `event.scheduleId` identifies the run
+as well. The handler skips a run whose schedule isn't the task's current
+`snoozeId`, and clears `snoozeId` when it brings the task back:
+
+```js
+async function unsnooze ({ event, ctx, user }) {
+    const key = `task:${ctx.taskId}`;
+    const task = await user.kv.get(key);
+    if (!task || task.snoozeId !== event.scheduleId) return;
+
+    await user.notifications.notify({
+        title: 'Back on your list',
+        text: task.title,
+        target: `/tasks/${ctx.taskId}`,
+        group: `task:${ctx.taskId}`,
+    });
+    await user.kv.update(key, { snoozedUntil: null, snoozeId: null });
+}
+```
+
+A few details:
+
+- A retry of a run that finished finds `snoozeId` cleared, and does nothing.
+- The handler sends the notification before it changes the task. If the run
+  fails between the two, the retry sends the notification again. A later
+  notification with the same `group` replaces the earlier one, so the user
+  sees it once.
+- Two attempts that overlap, such as a run still going past its 30 seconds
+  when its retry starts, can both pass the check. Both write the same values,
+  and the `group` covers the notification.
+- The same check skips a run for a snooze the user has since cancelled or
+  moved, as the next sections show.
+
+Publish the handler again after the change. A deploy script runs in a new
+process each time, so its SDK hasn't seen what's published. Call
+[`handlers.list()`](/Events/handlers/) before publishing changed code, or
+[`publish()`](/Events/handlers/) rejects with `events_handler_conflict`:
+
+```js
+await puter.events.handlers.list({ appUid: app.uid });
+await puter.events.handlers.publish('unsnooze', unsnooze, { appUid: app.uid });
+```
+
+## Bring a Task Back Early
+
+When the user brings a task back before its time, clear the snooze and end
+the schedule with [`puter.events.removeSchedule()`](/Events/removeSchedule/):
+
+```js
+async function endSchedule (scheduleId) {
+    try {
+        await puter.events.removeSchedule(scheduleId);
+    } catch (error) {
+        // It already ran, or was already removed
+        if (error.code !== 'events_schedule_not_found') throw error;
+    }
+}
+
+async function unsnoozeNow (taskId) {
+    const key = `task:${taskId}`;
+    const task = await puter.kv.get(key);
+    if (!task?.snoozeId) return;
+
+    await puter.kv.update(key, { snoozedUntil: null, snoozeId: null });
+    await endSchedule(task.snoozeId);
+}
+```
+
+A one-time schedule ends once its run is done. Removing a schedule that has
+ended, or one that was already removed, rejects with
+`events_schedule_not_found`, so `endSchedule()` treats that error as done.
+
+The code clears the task before it removes the schedule. Removing a schedule
+doesn't stop a run that already started, and that run finds `snoozeId`
+cleared and does nothing. A removal that fails, for example with a network
+error, leaves a schedule that runs once and changes nothing. Removing it still
+matters, since a schedule counts toward the user's limit until it runs or is
+removed.
+
+## Change the Snooze Time
+
+A schedule can't be edited. To move a snooze, create a new schedule and
+remove the old one. This version of `snooze()` does both, so it works for a
+new snooze and for moving one:
+
+```js
+async function snooze (taskId, until) {
+    const key = `task:${taskId}`;
+    const previous = await puter.kv.get(key);
+
+    const schedule = await puter.events.onSchedule({
+        at: until,
+        handlerName: 'unsnooze',
+        context: { taskId },
+    });
+    await puter.kv.update(key, {
+        snoozedUntil: schedule.startAt,
+        snoozeId: schedule.scheduleId,
+    });
+
+    if (previous?.snoozeId) await endSchedule(previous.snoozeId);
+}
+```
+
+The task points at the new schedule before the old one is removed. If the old
+one runs anyway, it finds a different `snoozeId` and does nothing.
+
+## Repeat a Few Times, Then Stop
+
+Some changes repeat a set number of times. For example, a task such as "Take
+your medicine" comes back every day for 5 days. Give
+[`onSchedule()`](/Events/onSchedule/) an `every` interval and `maxRuns`, and
+the schedule ends on its own after its last run.
+
+Each run of a repeating schedule has its own `event.id`, so the handler keeps
+the id of the last run it handled and skips a retry of that run. Publish it as
+the app's developer, like `unsnooze`:
+
+```js
+async function reopenTask ({ event, ctx, user }) {
+    const key = `task:${ctx.taskId}`;
+    const task = await user.kv.get(key);
+    if (!task || task.repeatId !== event.scheduleId || task.lastRunId === event.id) return;
+
+    await user.notifications.notify({
+        title: 'Back on your list',
+        text: task.title,
+        target: `/tasks/${ctx.taskId}`,
+        group: `task:${ctx.taskId}`,
+    });
+    await user.kv.update(key, { done: false, lastRunId: event.id });
+}
+
+await puter.events.handlers.publish('reopenTask', reopenTask, { appUid: app.uid });
+```
+
+In the app, create the schedule and keep its id with the task, as `snooze()`
+does:
+
+```js
+async function repeatTask (taskId, every, times) {
+    const schedule = await puter.events.onSchedule({
+        every,
+        maxRuns: times,
+        handlerName: 'reopenTask',
+        context: { taskId },
+    });
+    await puter.kv.update(`task:${taskId}`, { repeatId: schedule.scheduleId });
+}
+
+async function stopRepeating (taskId) {
+    const key = `task:${taskId}`;
+    const task = await puter.kv.get(key);
+    if (!task?.repeatId) return;
+
+    await puter.kv.update(key, { repeatId: null });
+    await endSchedule(task.repeatId);
+}
+
+if (allowed) await repeatTask('43', '1d', 5);    // once a day, 5 times
+```
+
+The first run is one interval after the call, so this reopens the task
+tomorrow at this time and on the 4 days after that. To pick the first run,
+pass `startAt`. The `maxRuns` option counts the times the schedule was due,
+so a run skipped while the schedule was suspended still counts. To stop at a
+time instead, pass `expiresAt`, the last time the schedule may run. With both,
+whichever comes first ends the schedule.
+
+An `every` interval is a fixed length of time, so a daily run moves by an hour
+in local time when the clocks change. To run at the same local time each day,
+use `cron` and `tz`, as
+[Run Code on a Schedule](/recipes/run-code-on-a-schedule/#create-the-schedule-once-per-user)
+does. The `maxRuns` option works with `cron` too.
+
+A user can hold fewer repeating schedules than one-time ones for your app.
+Past the limit, [`onSchedule()`](/Events/onSchedule/) rejects with
+`events_schedule_limit`. See
+[Rate Limits and Quotas](/rate-limits-and-quotas/#schedules).
+
+## Full Code
+
+The recipe has two programs. The publish script runs as the app's developer,
+from a deploy script or your browser console:
+
+```js
+const app = await puter.apps.get('task-list');    // your app's name
+
+async function unsnooze ({ event, ctx, user }) {
+    const key = `task:${ctx.taskId}`;
+    const task = await user.kv.get(key);
+    if (!task || task.snoozeId !== event.scheduleId) return;
+
+    await user.notifications.notify({
+        title: 'Back on your list',
+        text: task.title,
+        target: `/tasks/${ctx.taskId}`,
+        group: `task:${ctx.taskId}`,
+    });
+    await user.kv.update(key, { snoozedUntil: null, snoozeId: null });
+}
+
+async function reopenTask ({ event, ctx, user }) {
+    const key = `task:${ctx.taskId}`;
+    const task = await user.kv.get(key);
+    if (!task || task.repeatId !== event.scheduleId || task.lastRunId === event.id) return;
+
+    await user.notifications.notify({
+        title: 'Back on your list',
+        text: task.title,
+        target: `/tasks/${ctx.taskId}`,
+        group: `task:${ctx.taskId}`,
+    });
+    await user.kv.update(key, { done: false, lastRunId: event.id });
+}
+
+await puter.events.handlers.list({ appUid: app.uid });
+await puter.events.handlers.publish('unsnooze', unsnooze, { appUid: app.uid });
+await puter.events.handlers.publish('reopenTask', reopenTask, { appUid: app.uid });
+```
+
+The app code runs in your app, signed in as the user:
+
+```js
+async function endSchedule (scheduleId) {
+    try {
+        await puter.events.removeSchedule(scheduleId);
+    } catch (error) {
+        // It already ran, or was already removed
+        if (error.code !== 'events_schedule_not_found') throw error;
+    }
+}
+
+async function snooze (taskId, until) {
+    const key = `task:${taskId}`;
+    const previous = await puter.kv.get(key);
+
+    const schedule = await puter.events.onSchedule({
+        at: until,
+        handlerName: 'unsnooze',
+        context: { taskId },
+    });
+    await puter.kv.update(key, {
+        snoozedUntil: schedule.startAt,
+        snoozeId: schedule.scheduleId,
+    });
+
+    if (previous?.snoozeId) await endSchedule(previous.snoozeId);
+}
+
+async function unsnoozeNow (taskId) {
+    const key = `task:${taskId}`;
+    const task = await puter.kv.get(key);
+    if (!task?.snoozeId) return;
+
+    await puter.kv.update(key, { snoozedUntil: null, snoozeId: null });
+    await endSchedule(task.snoozeId);
+}
+
+async function repeatTask (taskId, every, times) {
+    const schedule = await puter.events.onSchedule({
+        every,
+        maxRuns: times,
+        handlerName: 'reopenTask',
+        context: { taskId },
+    });
+    await puter.kv.update(`task:${taskId}`, { repeatId: schedule.scheduleId });
+}
+
+async function stopRepeating (taskId) {
+    const key = `task:${taskId}`;
+    const task = await puter.kv.get(key);
+    if (!task?.repeatId) return;
+
+    await puter.kv.update(key, { repeatId: null });
+    await endSchedule(task.repeatId);
+}
+
+const allowed = await puter.perms.request('backgroundEvents');
+if (allowed) {
+    await snooze('42', new Date(Date.now() + 3 * 60 * 60 * 1000));    // three hours from now
+    await repeatTask('43', '1d', 5);    // once a day, 5 times
+}
+```

src/docs/src/recipes/poll-an-api-in-the-background.md

@@ -0,0 +1,242 @@
+---
+title: Poll an API in the Background
+description: "Learn how to check an outside service for new data on a schedule with Puter.js, so your app keeps each user's copy up to date even while it's closed."
+tags: [events, kv, net]
+order: 71
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+<div class="info">Schedules are available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+Some services can't tell your app when something changes, so your app has to
+ask them every so often. For example, importing new orders from a store,
+syncing a user's bookmarks from another service, or pulling the latest
+readings from a sensor. On other platforms you'd run a cron job on a server
+that calls the API for every user.
+
+In Puter, each user gets a schedule that runs your handler in your app's
+[events worker](/Events/workers/), every 15 minutes in this recipe, even
+while your app is closed. The handler asks the API for what changed since
+its last run, and saves it in the user's [key-value store](/KV/).
+[Run Code on a Schedule](/recipes/run-code-on-a-schedule/) covers handlers,
+permission and creating one schedule per user. This recipe adds the two
+things polling needs, a cursor and a way to finish a large batch.
+
+The examples call an API that returns the changes after a cursor, a page at a
+time:
+
+```
+GET https://api.example.com/changes?after=<cursor>
+→ { "items": [{ "id": "o-1", ... }, ...], "next": "<cursor>", "hasMore": true }
+```
+
+Most APIs that list changes work like this, with a cursor, a page token or a
+timestamp.
+
+## Publish the Handler
+
+The handler keeps its cursor in the user's key-value store, so each run picks
+up where the last one stopped. Publish it as the app's developer, from a
+deploy script or your browser console:
+
+```js
+const app = await puter.apps.get('order-sync');    // your app's name
+
+async function syncChanges ({ ctx, user, fetch }) {
+    const cursor = (await user.kv.get('sync:cursor')) ?? '';
+    const res = await fetch(`${ctx.apiUrl}/changes?after=${encodeURIComponent(cursor)}`, {
+        headers: { Authorization: `Bearer ${ctx.apiKey}` },
+    });
+    if (!res.ok) return;
+
+    const page = await res.json();
+    for (const item of page.items) {
+        await user.kv.set(`order:${item.id}`, item);
+    }
+    await user.kv.set('sync:cursor', page.next);
+}
+
+await puter.events.handlers.publish('syncChanges', syncChanges, { appUid: app.uid });
+```
+
+The handler gets `ctx`, the `context` the schedule was created with, and
+`fetch`, which reaches outside services from the events worker. A few
+details:
+
+- Each item is saved under its own id, so saving it twice is harmless. That
+  matters because a run can happen more than once.
+- The cursor is saved after the items. If a run stops partway, the next run
+  fetches the same page again instead of skipping it.
+- When the API answers with an error, the handler returns without moving the
+  cursor, and the next run asks again. A handler that throws counts as a
+  failed run, and five failed runs in a row pause the schedule until you
+  publish the handler again, so an outage at the other service would pause
+  every user's sync.
+
+## Create the Schedule
+
+In the app, after the user connects their account on the other service,
+create the schedule with the API's address and the user's key:
+
+```js
+await puter.perms.request('backgroundEvents');
+
+await puter.events.onSchedule({
+    every: '15m',
+    handlerName: 'syncChanges',
+    context: { apiUrl: 'https://api.example.com', apiKey: userApiKey },
+});
+```
+
+Create it once per user. [Run Code on a Schedule](/recipes/run-code-on-a-schedule/#create-the-schedule-once-per-user)
+shows how to check [`listSchedules()`](/Events/listSchedules/) first.
+
+The `context` is stored with the schedule and can't be changed. When the
+user's key changes, remove the schedule with
+[`puter.events.removeSchedule()`](/Events/removeSchedule/) and create a new
+one. The [`listSchedules()`](/Events/listSchedules/) method shows the names
+of the `context` keys, never their values.
+
+The schedule runs every 15 minutes from when it's created, which is as often
+as a free plan allows. A paid plan can run one every minute. See
+[Rate Limits and Quotas](/rate-limits-and-quotas/#schedules).
+
+## Finish a Large Batch
+
+A run has 30 seconds. After a long break, for example on the first sync, the
+API may have more pages than one run can fetch. The handler keeps fetching
+while there's time, and when pages are left over it schedules one extra run a
+minute later, with [`user.events.onSchedule()`](/Events/onSchedule/) and `at`:
+
+```js
+async function syncChanges ({ ctx, user, fetch }) {
+    const stopAt = Date.now() + 20 * 1000;
+    let cursor = (await user.kv.get('sync:cursor')) ?? '';
+    let hasMore = true;
+
+    while (hasMore && Date.now() < stopAt) {
+        let res;
+        try {
+            res = await fetch(`${ctx.apiUrl}/changes?after=${encodeURIComponent(cursor)}&limit=100`, {
+                headers: { Authorization: `Bearer ${ctx.apiKey}` },
+                signal: AbortSignal.timeout(5000),
+            });
+        } catch {
+            return;    // too slow or unreachable, so the next run asks again
+        }
+        if (!res.ok) return;
+
+        const page = await res.json();
+        if (page.items.length > 0) {
+            await user.kv.set(page.items.map((item) => ({ key: `order:${item.id}`, value: item })));
+        }
+        cursor = page.next;
+        hasMore = page.hasMore;
+        await user.kv.set('sync:cursor', cursor);
+    }
+
+    if (hasMore && !ctx.followUp) {
+        await user.events.onSchedule({
+            at: Date.now() + 60 * 1000,
+            handlerName: 'syncChanges',
+            context: { ...ctx, followUp: true },
+        });
+    }
+}
+```
+
+A few details:
+
+- The loop stops starting new pages after 20 seconds. Each page is at most
+  100 items (`limit`, if the API takes one), its request gives up after 5
+  seconds, and its items are saved in one call, so the last page started
+  still fits inside the 30. Without those bounds, one slow request or a long
+  run of writes could use up the 30 seconds before the cursor is saved, and
+  every run would fail at the same page.
+- The extra run is a one-time schedule with the same handler and the same
+  `context`, plus `followUp: true`. It continues from the saved cursor and
+  ends on its own after it runs.
+- An extra run doesn't schedule another one. Whatever it leaves over, the
+  next 15-minute run picks up. A schedule created from a handler counts as
+  one step deeper in a [handler chain](/Events/onPersistent/#handlers-that-trigger-handlers),
+  so extra runs that each scheduled the next would stop after a few steps.
+- The extra run and a regular run can overlap. Both start from the saved
+  cursor, so they can fetch the same page twice, and whichever saves last
+  wins, even when the other got further. A cursor that moves back only means
+  pages are fetched again. An item the slower run saves can be older than the
+  one it replaces until a later run fetches it again. If that matters, keep the
+  API's `updatedAt` on each item and skip writing one older than what's
+  stored. The key-value store has no compare-and-set, so the handler has to
+  tolerate the rare overlap rather than prevent it.
+
+Publish the handler again after the change. A deploy script runs in a new
+process each time, so its SDK hasn't seen what's published. Call
+[`handlers.list()`](/Events/handlers/) before publishing changed code, or
+[`publish()`](/Events/handlers/) rejects with `events_handler_conflict`:
+
+```js
+await puter.events.handlers.list({ appUid: app.uid });
+await puter.events.handlers.publish('syncChanges', syncChanges, { appUid: app.uid });
+```
+
+## Full Code
+
+The publish script runs as the app's developer, from a deploy script or your
+browser console:
+
+```js
+const app = await puter.apps.get('order-sync');    // your app's name
+
+async function syncChanges ({ ctx, user, fetch }) {
+    const stopAt = Date.now() + 20 * 1000;
+    let cursor = (await user.kv.get('sync:cursor')) ?? '';
+    let hasMore = true;
+
+    while (hasMore && Date.now() < stopAt) {
+        let res;
+        try {
+            res = await fetch(`${ctx.apiUrl}/changes?after=${encodeURIComponent(cursor)}&limit=100`, {
+                headers: { Authorization: `Bearer ${ctx.apiKey}` },
+                signal: AbortSignal.timeout(5000),
+            });
+        } catch {
+            return;    // too slow or unreachable, so the next run asks again
+        }
+        if (!res.ok) return;
+
+        const page = await res.json();
+        if (page.items.length > 0) {
+            await user.kv.set(page.items.map((item) => ({ key: `order:${item.id}`, value: item })));
+        }
+        cursor = page.next;
+        hasMore = page.hasMore;
+        await user.kv.set('sync:cursor', cursor);
+    }
+
+    if (hasMore && !ctx.followUp) {
+        await user.events.onSchedule({
+            at: Date.now() + 60 * 1000,
+            handlerName: 'syncChanges',
+            context: { ...ctx, followUp: true },
+        });
+    }
+}
+
+await puter.events.handlers.list({ appUid: app.uid });
+await puter.events.handlers.publish('syncChanges', syncChanges, { appUid: app.uid });
+```
+
+The app code runs in your app, signed in as the user, once they've connected
+the other service. Here `userApiKey` is the key your app got for them:
+
+```js
+const allowed = await puter.perms.request('backgroundEvents');
+if (allowed) {
+    await puter.events.onSchedule({
+        every: '15m',
+        handlerName: 'syncChanges',
+        context: { apiUrl: 'https://api.example.com', apiKey: userApiKey },
+    });
+}
+```

src/docs/src/recipes/remind-users-later.md

@@ -0,0 +1,229 @@
+---
+title: Remind Users Later
+description: "Learn how to send a user a reminder at a time they pick with Puter.js, and cancel it if they no longer need it, even while your app is closed."
+tags: [notifications, events, kv]
+order: 69
+---
+
+<div class="info">Sending later with <code>sendAt</code> is available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+A "remind me later" button, a reminder before a meeting, or a note that a
+trial is about to end all need a notification that goes out at a set time,
+often when nobody has your app open. On other platforms you'd store the
+reminder in a database and run a job that checks it every minute.
+
+With Puter.js, you pass the time to
+[`puter.notifications.notify()`](/Notifications/notify/) as `sendAt`. Puter
+stores the notification and sends it at that time. Your app doesn't need to
+be open, and you don't need a server or a handler.
+
+The examples build reminders for a task list app. The user picks a task and a
+time, and gets a notification about it then.
+
+## Send a Notification Later
+
+Call [`notify()`](/Notifications/notify/) with `sendAt`:
+
+```js
+const reminder = await puter.notifications.notify({
+    title: 'Reminder: Call the dentist',
+    target: '/tasks/42',
+    sendAt: Date.now() + 2 * 60 * 60 * 1000,    // two hours from now
+});
+```
+
+The `sendAt` option takes a `Date`, an ISO-8601 string with a time zone such
+as `'2026-11-14T09:00:00-05:00'`, or milliseconds since the epoch. It can be
+up to 366 days ahead.
+
+The call resolves to `{ scheduleId, sendAt }`. A `sendAt` within 5 seconds,
+or already past, isn't stored. The notification goes out at once, and the call
+resolves as it would without `sendAt`, with the notification's `uid` (a
+string). Check which one you got before keeping a `scheduleId`.
+
+Puter usually sends the stored notification within 15 seconds of `sendAt`, as
+if your app called [`notify()`](/Notifications/notify/) then. It reaches every
+open page of your app, and goes out as a push when no page is open and the
+user turned pushes on.
+[Reach a User When Your App Is Closed](/recipes/reach-a-user-when-your-app-is-closed/)
+shows how to set pushes up, and `target` is the page the user lands on when
+they tap it.
+
+## Cancel a Reminder
+
+When the user finishes the task first, the reminder shouldn't go out. Keep the
+`scheduleId` with the task, in the app's
+[key-value store](/KV/):
+
+```js
+async function remindLater (taskId, title, sendAt) {
+    const reminder = await puter.notifications.notify({
+        title: `Reminder: ${title}`,
+        target: `/tasks/${taskId}`,
+        sendAt,
+    });
+    // A time within 5 seconds, or past, is sent at once, so there's nothing to cancel.
+    if (typeof reminder !== 'object' || !reminder) return;
+    await puter.kv.set(`reminder:${taskId}`, reminder.scheduleId);
+}
+```
+
+Then cancel it with
+[`puter.notifications.cancel()`](/Notifications/cancel/) when the task is
+done:
+
+```js
+async function cancelReminder (taskId) {
+    const scheduleId = await puter.kv.get(`reminder:${taskId}`);
+    if (!scheduleId) return;
+
+    try {
+        await puter.notifications.cancel(scheduleId);
+    } catch (error) {
+        // Already sent, or going out right now
+        if (error.code !== 'events_schedule_not_found' && error.code !== 'events_schedule_sending') throw error;
+    }
+    await puter.kv.del(`reminder:${taskId}`);
+}
+```
+
+A reminder that was already sent, or already cancelled, rejects with
+`events_schedule_not_found`. One that's going out right now rejects with
+`events_schedule_sending`. Either way it's too late to stop it, so the code
+removes the stored id and moves on.
+
+To move a reminder to a new time, cancel it and call `remindLater()` again.
+
+## Show the Reminders That Are Waiting
+
+To list the notifications your app has waiting for the user, along with any
+[schedules](/Events/onSchedule/) it created, use
+[`puter.events.listSchedules()`](/Events/listSchedules/). A stored
+notification has `kind: 'deferred'`, and `nextRunAt` is when it goes out:
+
+```js
+async function listReminders () {
+    const waiting = [];
+    let cursor = null;
+    do {
+        const page = await puter.events.listSchedules({ cursor });
+        for (const schedule of page.items) {
+            if (schedule.kind === 'deferred' && schedule.state === 'active') {
+                waiting.push({ scheduleId: schedule.scheduleId, sendAt: schedule.nextRunAt });
+            }
+        }
+        cursor = page.cursor ?? null;
+    } while (cursor);
+    return waiting;
+}
+```
+
+The list doesn't include the notification's title, so match each
+`scheduleId` with the ids stored by `remindLater()` to show which task it's
+for.
+
+There's a limit on how many notifications your app can have waiting for one
+user. Past it, [`notify()`](/Notifications/notify/) rejects with
+`events_schedule_limit`.
+See [Rate Limits and Quotas](/rate-limits-and-quotas/#schedules).
+
+## Decide When the Reminder Is Due
+
+A notification sent with `sendAt` is fixed when you store it. To decide at
+the last moment, for example to skip the reminder when the task is already
+done, run a handler at that time instead with
+[`puter.events.onSchedule()`](/Events/onSchedule/) and `at`. The handler
+checks the task and sends the notification itself.
+
+Publish the handler once, as the app's developer, from a deploy script or
+your browser console. [Run Code on a Schedule](/recipes/run-code-on-a-schedule/)
+explains handlers and the `user` object they get:
+
+```js
+const app = await puter.apps.get('task-list');    // your app's name
+
+async function remindIfOpen ({ event, ctx, user }) {
+    const sentKey = `reminded:${event.id}`;
+    if (await user.kv.get(sentKey)) return;
+
+    const task = await user.kv.get(`task:${ctx.taskId}`);
+    if (task && !task.done) {
+        await user.notifications.notify({
+            title: `Reminder: ${task.title}`,
+            target: `/tasks/${ctx.taskId}`,
+        });
+    }
+
+    await user.kv.set(sentKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}
+
+await puter.events.handlers.publish('remindIfOpen', remindIfOpen, { appUid: app.uid });
+```
+
+Then, in the app, create a one-time schedule for the task:
+
+```js
+await puter.perms.request('backgroundEvents');
+
+const schedule = await puter.events.onSchedule({
+    at: Date.now() + 2 * 60 * 60 * 1000,
+    handlerName: 'remindIfOpen',
+    context: { taskId: '42' },
+});
+```
+
+The handler gets `context` as `ctx`, so it knows which task to check. A run
+can happen more than once, so the handler saves `event.id` after sending and
+skips it the next time. Running a handler needs the user's
+`events:background` permission, which a plain `sendAt` doesn't.
+
+Cancel it the same way as a stored notification, with
+[`puter.events.removeSchedule()`](/Events/removeSchedule/) and
+`schedule.scheduleId`.
+[Change Data at a Set Time](/recipes/change-data-at-a-set-time/) shows how to
+cancel or move a one-time schedule, including one that has already run.
+
+## Full Code
+
+The app code for reminders sent with `sendAt`, signed in as the user:
+
+```js
+async function remindLater (taskId, title, sendAt) {
+    const reminder = await puter.notifications.notify({
+        title: `Reminder: ${title}`,
+        target: `/tasks/${taskId}`,
+        sendAt,
+    });
+    // A time within 5 seconds, or past, is sent at once, so there's nothing to cancel.
+    if (typeof reminder !== 'object' || !reminder) return;
+    await puter.kv.set(`reminder:${taskId}`, reminder.scheduleId);
+}
+
+async function cancelReminder (taskId) {
+    const scheduleId = await puter.kv.get(`reminder:${taskId}`);
+    if (!scheduleId) return;
+
+    try {
+        await puter.notifications.cancel(scheduleId);
+    } catch (error) {
+        // Already sent, or going out right now
+        if (error.code !== 'events_schedule_not_found' && error.code !== 'events_schedule_sending') throw error;
+    }
+    await puter.kv.del(`reminder:${taskId}`);
+}
+
+async function listReminders () {
+    const waiting = [];
+    let cursor = null;
+    do {
+        const page = await puter.events.listSchedules({ cursor });
+        for (const schedule of page.items) {
+            if (schedule.kind === 'deferred' && schedule.state === 'active') {
+                waiting.push({ scheduleId: schedule.scheduleId, sendAt: schedule.nextRunAt });
+            }
+        }
+        cursor = page.cursor ?? null;
+    } while (cursor);
+    return waiting;
+}
+```

src/docs/src/recipes/run-code-on-a-schedule.md

@@ -0,0 +1,340 @@
+---
+title: Run Code on a Schedule
+description: "Learn how to run your own code on a schedule with Puter.js, like a cron job, so your app can send each user a morning summary in their own time zone, even while the app is closed."
+tags: [events, notifications, kv]
+order: 68
+---
+
+<div class="info">The Events API is in beta. Event shapes, limits, and behavior may change between releases.</div>
+
+<div class="info">Schedules are available on puter.com. A self-hosted Puter server answers <code>events_disabled</code>.</div>
+
+Some work happens on a clock instead of in response to something the user
+did. For example, sending a summary every weekday morning, clearing out old
+data every night, or checking a feed every hour. On other platforms you'd use
+a cron job or a scheduled cloud function.
+
+In Puter, you write a function, publish it once, and give each user a
+schedule that runs it. Puter runs it in your app's
+[events worker](/Events/workers/), a serverless function that belongs to your
+app, at the times the schedule says. You don't need to run a server.
+
+[Run Code in the Background](/recipes/run-code-in-the-background/) runs a
+function when a user's files change. This recipe runs one at set times
+instead, with [`puter.events.onSchedule()`](/Events/onSchedule/). The examples
+build a task list app that sends each user a notification at 8:00 every
+weekday morning, in their own time zone, with how many tasks they have open.
+
+## Publish the Handler
+
+The function a schedule runs is called a handler. Publish it with
+[`puter.events.handlers.publish()`](/Events/handlers/). This is a developer
+step, like a deploy, and it only works for the account that owns the app. Run
+it from a deploy script or your browser console:
+
+```js
+const app = await puter.apps.get('task-list');    // your app's name
+
+async function sendSummary ({ user }) {
+    const tasks = (await user.kv.get('tasks')) ?? [];
+    const open = tasks.filter((task) => !task.done);
+    if (open.length === 0) return;
+
+    await user.notifications.notify({
+        title: 'Good morning',
+        text: `You have ${open.length} open tasks today.`,
+        target: '/tasks',
+        group: 'morning-summary',
+    });
+}
+
+await puter.events.handlers.publish('sendSummary', sendSummary, { appUid: app.uid });
+```
+
+The handler gets one object as its argument. This one uses `user`, a `puter`
+object signed in as the user the schedule belongs to. It can do whatever your
+app can do for that user, like `user.kv` and `user.notifications`. It's valid
+for 15 minutes, so use it only during the run.
+
+Puter stores the handler's code and runs it later in the events worker, so
+the handler can only use its parameters, its own variables, and standard
+globals like `fetch`, `JSON` and `Math`. It uses `user` instead of `puter`.
+The [`publish()`](/Events/handlers/) method checks this, and fails with
+`events_handler_free_variable` if the handler uses a name it doesn't define.
+
+The notification reaches every open page of your app, and goes out as a push
+when none is open and the user turned pushes on.
+[Reach a User When Your App Is Closed](/recipes/reach-a-user-when-your-app-is-closed/) shows how
+to set pushes up. The `group` makes each morning's summary replace the one
+before it.
+
+## Ask for Permission
+
+A schedule runs code while the user isn't there, so it needs the user's
+permission. Ask from your app with [`puter.perms.request()`](/Perms/request/):
+
+```js
+const allowed = await puter.perms.request('backgroundEvents');
+```
+
+It resolves to `true` or `false`. If the permission is already granted, the
+user doesn't see a prompt. Without it,
+[`onSchedule()`](/Events/onSchedule/) fails with
+`events_background_consent_required`.
+
+If the user revokes the permission later, every schedule your app holds for
+them ends. Granting it again doesn't bring them back, so your app has to
+create them again. Schedules also need a Puter account. For a guest account,
+[`onSchedule()`](/Events/onSchedule/) fails with
+`events_durable_requires_account`.
+
+## Create the Schedule Once per User
+
+The schedule says when to run the handler. Here it's a `cron` expression with
+the user's time zone:
+
+```js
+const options = {
+    cron: '0 8 * * 1-5',
+    tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
+    handlerName: 'sendSummary',
+};
+```
+
+The five fields of `0 8 * * 1-5` are the minute, the hour, the day of the
+month, the month and the day of the week, so it means 8:00 on Monday to
+Friday. The `tz` option is the browser's time zone, such as
+`'America/New_York'`. The schedule follows that zone's daylight saving time,
+so the summary arrives at 8:00 local time all year. The
+[`onSchedule()`](/Events/onSchedule/#cron-expressions) page lists the cron
+syntax it accepts.
+
+Every [`onSchedule()`](/Events/onSchedule/) call creates a new schedule, even
+with the same options. If you call it each time the app opens, the user gets
+one more summary each morning for every time they opened it. Check
+[`puter.events.listSchedules()`](/Events/listSchedules/) first:
+
+```js
+async function findSchedule () {
+    let cursor = null;
+    do {
+        const page = await puter.events.listSchedules({ cursor });
+        const found = page.items.find((schedule) =>
+            schedule.handlerName === options.handlerName && schedule.state !== 'ended');
+        if (found) return found;
+        cursor = page.cursor ?? null;
+    } while (cursor);
+    return null;
+}
+
+async function startMorningSummary () {
+    if (await findSchedule()) return;
+    await puter.events.onSchedule(options);
+}
+```
+
+Call `startMorningSummary()` when the app opens, after the user has given
+permission. A few details:
+
+- The [`listSchedules()`](/Events/listSchedules/) method returns one page at a
+  time, and only your app's schedules for this user. The loop follows
+  `cursor` until there are no more pages.
+- Ended schedules stay in the list for 7 days, so the code skips ones with
+  `state: 'ended'`. That includes a schedule that ended because the user
+  revoked the permission, so the app creates a new one once it's granted again.
+- A suspended schedule has `state: 'suspended'`. It resumes on its own, for
+  example when the user adds credit, so the code keeps it.
+
+If two tabs open at the same moment, both might create a schedule. If
+[`listSchedules()`](/Events/listSchedules/) finds more than one, keep one and
+remove the rest with
+[`puter.events.removeSchedule()`](/Events/removeSchedule/).
+
+## Skip Summaries Already Sent
+
+Each run happens at least once. A run that times out, or fails with a network
+error, is retried, and a retry can repeat work the first attempt already did.
+Every attempt at the same run has the same `event.id`, so the handler saves
+it after sending and skips ids it has already sent. The marker key expires
+after a week, so old markers are removed from the user's key-value store on
+their own:
+
+```js
+async function sendSummary ({ event, user }) {
+    const sentKey = `summary-sent:${event.id}`;
+    if (await user.kv.get(sentKey)) return;
+
+    const tasks = (await user.kv.get('tasks')) ?? [];
+    const open = tasks.filter((task) => !task.done);
+    if (open.length > 0) {
+        await user.notifications.notify({
+            title: 'Good morning',
+            text: `You have ${open.length} open tasks today.`,
+            target: '/tasks',
+            group: 'morning-summary',
+        });
+    }
+
+    await user.kv.set(sentKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}
+```
+
+This skips a retry of a run that already finished. It doesn't rule out every
+repeat. Two attempts that overlap, such as a run still going past its 30
+seconds when its retry starts, can both pass the check, and a run that stops
+after sending but before saving the key sends again when it's retried. Here a
+repeat carries the same `group`, so it replaces the summary that's showing
+instead of adding a second one. For work that must happen exactly once, such as
+a payment, make the call itself safe to repeat, for example by passing
+`event.id` as the idempotency key of the service you call.
+
+Publish the handler again after the change. The schedules already created use
+the new code from their next run.
+
+A deploy script runs in a new process each time, so its SDK hasn't seen what's
+published. Call [`handlers.list()`](/Events/handlers/) before publishing changed
+code, or [`publish()`](/Events/handlers/) rejects with
+`events_handler_conflict`:
+
+```js
+await puter.events.handlers.list({ appUid: app.uid });
+await puter.events.handlers.publish('sendSummary', sendSummary, { appUid: app.uid });
+```
+
+## Handle Mornings That Were Missed
+
+A schedule can fall behind. For example, if the handler failed 5 times in a
+row, Puter pauses the schedule until you publish a fix. When it resumes, it
+doesn't send every summary it missed. It runs the next one, and `event.missed`
+says how many runs were skipped. The summary can say so:
+
+```js
+async function sendSummary ({ event, user }) {
+    const sentKey = `summary-sent:${event.id}`;
+    if (await user.kv.get(sentKey)) return;
+
+    const tasks = (await user.kv.get('tasks')) ?? [];
+    const open = tasks.filter((task) => !task.done);
+    const skipped = event.missed > 0 ? ` ${event.missed} earlier summaries were skipped.` : '';
+    if (open.length > 0) {
+        await user.notifications.notify({
+            title: 'Good morning',
+            text: `You have ${open.length} open tasks today.${skipped}`,
+            target: '/tasks',
+            group: 'morning-summary',
+        });
+    }
+
+    await user.kv.set(sentKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}
+```
+
+The run's `event.scheduledFor` is the time it was due, in milliseconds since
+the epoch. A run usually starts within a minute of it.
+
+## Let the User Change the Time
+
+A schedule can't be edited. To move the summary to another time or time zone,
+remove the schedule and create a new one:
+
+```js
+async function changeSummaryTime (hour, timeZone) {
+    const existing = await findSchedule();
+    if (existing) await puter.events.removeSchedule(existing.scheduleId);
+
+    options.cron = `0 ${hour} * * 1-5`;
+    options.tz = timeZone;
+    await puter.events.onSchedule(options);
+}
+```
+
+To turn the summary off, call
+[`puter.events.removeSchedule()`](/Events/removeSchedule/) with the
+`scheduleId` from `findSchedule()`.
+
+## Check on Schedules
+
+A suspended schedule still exists, and
+[`listSchedules()`](/Events/listSchedules/) shows why it's paused in
+`suspendedReason`:
+
+| `suspendedReason` | What happened | What to do |
+| --- | --- | --- |
+| `handler_not_found` | The handler was removed. | Publish it again. The schedule resumes. |
+| `failures` | Five runs in a row failed. | Fix the handler and publish it again. The schedule resumes. |
+| `no_credit` | The user ran out of credit. | Nothing. It resumes when they add credit. |
+
+Each run is billed to the user under the [User-Pays Model](/user-pays-model/),
+as one event delivery plus the handler's worker usage. See
+[Rate Limits and Quotas](/rate-limits-and-quotas/#schedules) for how many
+schedules a user can have, and how often they can run.
+
+## Full Code
+
+The recipe has two programs. The publish script runs as the app's developer,
+from a deploy script or your browser console:
+
+```js
+const app = await puter.apps.get('task-list');    // your app's name
+
+async function sendSummary ({ event, user }) {
+    const sentKey = `summary-sent:${event.id}`;
+    if (await user.kv.get(sentKey)) return;
+
+    const tasks = (await user.kv.get('tasks')) ?? [];
+    const open = tasks.filter((task) => !task.done);
+    const skipped = event.missed > 0 ? ` ${event.missed} earlier summaries were skipped.` : '';
+    if (open.length > 0) {
+        await user.notifications.notify({
+            title: 'Good morning',
+            text: `You have ${open.length} open tasks today.${skipped}`,
+            target: '/tasks',
+            group: 'morning-summary',
+        });
+    }
+
+    await user.kv.set(sentKey, true, Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60);
+}
+
+await puter.events.handlers.list({ appUid: app.uid });
+await puter.events.handlers.publish('sendSummary', sendSummary, { appUid: app.uid });
+```
+
+The app code runs in your app, signed in as the user:
+
+```js
+const options = {
+    cron: '0 8 * * 1-5',
+    tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
+    handlerName: 'sendSummary',
+};
+
+async function findSchedule () {
+    let cursor = null;
+    do {
+        const page = await puter.events.listSchedules({ cursor });
+        const found = page.items.find((schedule) =>
+            schedule.handlerName === options.handlerName && schedule.state !== 'ended');
+        if (found) return found;
+        cursor = page.cursor ?? null;
+    } while (cursor);
+    return null;
+}
+
+async function startMorningSummary () {
+    if (await findSchedule()) return;
+    await puter.events.onSchedule(options);
+}
+
+async function changeSummaryTime (hour, timeZone) {
+    const existing = await findSchedule();
+    if (existing) await puter.events.removeSchedule(existing.scheduleId);
+
+    options.cron = `0 ${hour} * * 1-5`;
+    options.tz = timeZone;
+    await puter.events.onSchedule(options);
+}
+
+const allowed = await puter.perms.request('backgroundEvents');
+if (allowed) await startMorningSummary();
+```

src/docs/src/sidebar.js

@@ -515,6 +515,38 @@ let sidebar = [
                 source: '/Events/unsubscribe.md',
                 path: '/Events/unsubscribe',
             },
+            {
+                title: '<code>onSchedule()</code>',
+                page_title: '<code>puter.events.onSchedule()</code>',
+                title_tag: 'puter.events.onSchedule()',
+                icon: '/assets/img/function.svg',
+                source: '/Events/onSchedule.md',
+                path: '/Events/onSchedule',
+            },
+            {
+                title: '<code>listSchedules()</code>',
+                page_title: '<code>puter.events.listSchedules()</code>',
+                title_tag: 'puter.events.listSchedules()',
+                icon: '/assets/img/function.svg',
+                source: '/Events/listSchedules.md',
+                path: '/Events/listSchedules',
+            },
+            {
+                title: '<code>getSchedule()</code>',
+                page_title: '<code>puter.events.getSchedule()</code>',
+                title_tag: 'puter.events.getSchedule()',
+                icon: '/assets/img/function.svg',
+                source: '/Events/getSchedule.md',
+                path: '/Events/getSchedule',
+            },
+            {
+                title: '<code>removeSchedule()</code>',
+                page_title: '<code>puter.events.removeSchedule()</code>',
+                title_tag: 'puter.events.removeSchedule()',
+                icon: '/assets/img/function.svg',
+                source: '/Events/removeSchedule.md',
+                path: '/Events/removeSchedule',
+            },
             {
                 title: '<code>fetch()</code>',
                 page_title: '<code>puter.events.fetch()</code>',
@@ -571,6 +603,14 @@ let sidebar = [
                 source: '/Notifications/notify.md',
                 path: '/Notifications/notify',
             },
+            {
+                title: '<code>cancel()</code>',
+                page_title: '<code>puter.notifications.cancel()</code>',
+                title_tag: 'puter.notifications.cancel()',
+                icon: '/assets/img/function.svg',
+                source: '/Notifications/cancel.md',
+                path: '/Notifications/cancel',
+            },
             {
                 title: '<code>onNotification()</code>',
                 page_title: '<code>puter.notifications.onNotification()</code>',

src/puter-js/index.d.ts

@@ -139,11 +139,18 @@ export type {
     HandlerSummary,
     OnLocalOptions,
     OnPersistentOptions,
+    OnScheduleOptions,
     PersistentSubscription,
     PublishedHandler,
     PuterEvent,
     PuterKvEvent,
     PuterNotifEvent,
+    PuterScheduleEvent,
+    Schedule,
+    ScheduleDelivery,
+    ScheduleErrorCode,
+    ScheduleHandler,
+    ScheduleListOptions,
     UpdatePersistentOptions,
 } from './types/modules/events/types.js';
 export type { EventSubscription } from './types/modules/events/lib/subscription.js';
@@ -236,6 +243,7 @@ export type { PSocket, PTLSSocket } from './types/modules/networking/PSocket.js'
 // -- puter.notifications --
 export type {
     AppNotification,
+    DelayedNotificationOptions,
     NotificationClick,
     NotificationHandler,
     NotificationInput,
@@ -249,6 +257,7 @@ export type {
     PushPlatform,
     PushShown,
     PushStatus,
+    ScheduledNotification,
     ShareNotificationInput,
 } from './types/modules/notifications/types.js';
 

src/puter-js/src/lib/networkUtils.js

@@ -504,6 +504,30 @@ async function bodyForLog(xhr) {
 // can mean the request was half-applied upstream, so only a read replays.
 const RETRYABLE_STATUS = new Set([502, 503, 504]);
 
+// A 503 that names a feature this server doesn't have is final: asking again
+// gets the same answer.
+const FINAL_UNAVAILABLE_CODES = new Set(['events_disabled']);
+
+const isFinalUnavailable = (status, parsed) =>
+    status === 503 &&
+    [parsed?.code, parsed?.error?.code].some((code) =>
+        FINAL_UNAVAILABLE_CODES.has(code),
+    );
+
+// A 429 that refuses a quota (how many schedules, subscriptions, or handlers an
+// account holds) is final: it stays refused until something is removed.
+const FINAL_QUOTA_CODES = new Set([
+    'events_schedule_limit',
+    'events_schedule_budget',
+    'events_subscription_limit',
+    'events_handler_limit',
+]);
+
+const isQuotaRejection = (parsed) =>
+    [parsed?.code, parsed?.error?.code].some((code) =>
+        FINAL_QUOTA_CODES.has(code),
+    );
+
 // A rate/concurrency gate rejects in middleware, before the handler that would
 // have done the work — so nothing happened, and replaying is safe for a write
 // exactly as it is for a read. Treating this like the statuses above meant a
@@ -838,13 +862,26 @@ async function classifyRetry(outcome, ctx) {
     // gate rejection — any method, honors kill switch, fixed schedule (an
     // exponential one when it's waiting on the account's own requests).
     if (status === GATE_REJECT_STATUS) {
+        if (isQuotaRejection(parsed)) return null;
         return isCreditHoldRejection(parsed)
             ? creditHoldRetry(ctx)
             : gateRetry(ctx);
     }
 
     // transient status — read-safe only, honors kill switch, fixed schedule.
-    if (RETRYABLE_STATUS.has(status)) return transientRetry(ctx);
+    if (RETRYABLE_STATUS.has(status)) {
+        if (status === 503 && ctx.retrySafe) {
+            if (outcome.parsed === undefined) {
+                try {
+                    outcome.parsed = await bodyJson(xhr);
+                } catch {
+                    outcome.parsed = null;
+                }
+            }
+            if (isFinalUnavailable(status, outcome.parsed)) return null;
+        }
+        return transientRetry(ctx);
+    }
 
     return null;
 }

src/puter-js/src/lib/networkUtils.test.js

@@ -613,6 +613,47 @@ describe('transient retry', () => {
         vi.useRealTimers();
     });
 
+    // A server without a feature answers 503 `events_disabled` every time.
+    it.each([
+        [{ code: 'events_disabled', message: 'Schedules are not available' }],
+        [{ error: { code: 'events_disabled', message: 'Not available' } }],
+    ])(
+        'answers a GET whose 503 is a structured events_disabled at once (%j)',
+        async (body) => {
+            const xhrs = installFakeXHR(
+                sequence(respond({ status: 503, body })),
+            );
+            const resp = await fetchUrl('https://api.example/x'); // GET
+            expect(resp.status).toBe(503);
+            expect(xhrs.length).toBe(1);
+            expect(JSON.stringify(await resp.json())).toContain(
+                'events_disabled',
+            );
+        },
+    );
+
+    it.each([
+        [{ status: 503, body: { code: 'service_unavailable' } }],
+        [
+            {
+                status: 503,
+                body: 'upstream unavailable',
+                contentType: 'text/plain',
+            },
+        ],
+        [{ status: 502, body: { code: 'events_disabled' } }],
+    ])('keeps retrying any other 502/503 (%j)', async (first) => {
+        vi.useFakeTimers();
+        const xhrs = installFakeXHR(
+            sequence(respond(first), respond({ status: 200, body: { ok: 1 } })),
+        );
+        const p = fetchUrl('https://api.example/x');
+        await vi.advanceTimersByTimeAsync(60_000);
+        expect((await p).status).toBe(200);
+        expect(xhrs.length).toBe(2);
+        vi.useRealTimers();
+    });
+
     it('retry:false disables retry even for a GET', async () => {
         const xhrs = installFakeXHR(
             sequence(respond({ status: 503, body: {} })),
@@ -659,6 +700,40 @@ describe('transient retry', () => {
         vi.useRealTimers();
     });
 
+    // A quota refusal stays refused until the caller removes something.
+    it.each([
+        [{ code: 'events_schedule_limit', message: 'Limit reached' }],
+        [{ code: 'events_schedule_budget', message: 'Over budget' }],
+        [{ code: 'events_subscription_limit', message: 'Limit reached' }],
+        [{ code: 'events_handler_limit', message: 'Limit reached' }],
+        [{ error: { code: 'events_schedule_limit', message: 'Limit' } }],
+    ])('answers a 429 quota refusal at once (%j)', async (body) => {
+        const xhrs = installFakeXHR(sequence(respond({ status: 429, body })));
+        const resp = await fetchUrl('https://api.example/x', {
+            method: 'POST',
+        });
+        expect(resp.status).toBe(429);
+        expect(xhrs.length).toBe(1);
+    });
+
+    it('keeps retrying a too_many_requests 429', async () => {
+        vi.useFakeTimers();
+        const xhrs = installFakeXHR(
+            sequence(
+                respond({
+                    status: 429,
+                    body: { code: 'too_many_requests', message: 'Slow down' },
+                }),
+                respond({ status: 200, body: { ok: 1 } }),
+            ),
+        );
+        const p = fetchUrl('https://api.example/x', { method: 'POST' });
+        await vi.advanceTimersByTimeAsync(60_000);
+        expect((await p).status).toBe(200);
+        expect(xhrs.length).toBe(2);
+        vi.useRealTimers();
+    });
+
     // Waiting on the account's own requests to finish takes longer than a rate
     // window, so `credits_reserved` backs off exponentially past 2s.
     it('backs off exponentially on a credits_reserved 429', async () => {

src/puter-js/src/modules/events/index.js

@@ -6,6 +6,12 @@ import { fetch } from './fetch.js';
 import { list } from './list.js';
 import { onLocal } from './onLocal.js';
 import { onPersistent } from './onPersistent.js';
+import {
+    getSchedule,
+    listSchedules,
+    onSchedule,
+    removeSchedule,
+} from './schedules.js';
 import { unsubscribe } from './unsubscribe.js';
 import { update } from './update.js';
 
@@ -24,6 +30,8 @@ import { update } from './update.js';
  * `fetch()` is the other half: what happened while nothing was listening, read
  * from the subject's own store a page at a time.
  *
+ * `onSchedule()` runs a published handler on a timer rather than on a change.
+ *
  * A published handler set stands up an **events worker** per app —
  * `puter.events.workers` is where an owner sees and destroys them.
  *
@@ -35,12 +43,24 @@ export class EventsModule extends PuterModule {
     // The fields hold the unbound functions so they keep their full types
     // (`bind` erases overloads); the constructor rebinds them so destructured
     // calls (`const { onLocal } = puter.events`) work.
+    // A field typed without `this` keeps `puter.events.x()` compiling: its
+    // `this` names this class, which the public type omits members of.
     onLocal = onLocal;
+    /** @type {OmitThisParameter<typeof onPersistent>} */
     onPersistent = onPersistent;
+    /** @type {OmitThisParameter<typeof unsubscribe>} */
     unsubscribe = unsubscribe;
     update = update;
     list = list;
     fetch = fetch;
+    /** @type {OmitThisParameter<typeof onSchedule>} */
+    onSchedule = onSchedule;
+    /** @type {OmitThisParameter<typeof listSchedules>} */
+    listSchedules = listSchedules;
+    /** @type {OmitThisParameter<typeof getSchedule>} */
+    getSchedule = getSchedule;
+    /** @type {OmitThisParameter<typeof removeSchedule>} */
+    removeSchedule = removeSchedule;
 
     /** @param {Puter} puter */
     constructor (puter) {
@@ -68,6 +88,10 @@ export class EventsModule extends PuterModule {
             'update',
             'list',
             'fetch',
+            'onSchedule',
+            'listSchedules',
+            'getSchedule',
+            'removeSchedule',
         ] ) {
             methods[name] = methods[name].bind(this);
         }

src/puter-js/src/modules/events/schedules.js

@@ -0,0 +1,176 @@
+import { PuterJSError } from '../../lib/PuterJSError.js';
+import { request } from './lib/api.js';
+import { prepareHandler, serializeContext } from './lib/handlerSource.js';
+
+/** @typedef {import('./types.js').OnScheduleOptions} OnScheduleOptions */
+/** @typedef {import('./types.js').Schedule} Schedule */
+/** @typedef {import('./types.js').ScheduleListOptions} ScheduleListOptions */
+/** @typedef {import('../../lib/types.js').ListPage<Schedule>} SchedulePage */
+
+/** The options sent as they are; the server validates and rounds them. */
+const TIMING_OPTIONS = /** @type {const} */ ([
+    'every',
+    'cron',
+    'tz',
+    'at',
+    'startAt',
+    'expiresAt',
+    'maxRuns',
+]);
+
+const noSuchSchedule = () =>
+    new PuterJSError('No such schedule', 'events_schedule_not_found');
+
+/** @param {unknown} scheduleId */
+const assertScheduleId = (scheduleId) => {
+    if ( typeof scheduleId !== 'string' || scheduleId.trim().length === 0 ) {
+        throw noSuchSchedule();
+    }
+};
+
+/**
+ * Runs the app's published handler on a timer: every interval, on a cron
+ * expression in a time zone, or once at a set time. The schedule is stored
+ * against the account and runs with nobody present, in the app's events
+ * worker, never in this client.
+ *
+ * Give exactly one of `every`, `cron` or `at`. `handlerName` names the
+ * published handler each run calls; passing `handler` too checks that it is
+ * the source published under that name.
+ *
+ * `context` is evaluated **here, now** and delivered to every run as a frozen
+ * `ctx`. It can't be changed later: remove the schedule and create a new one.
+ *
+ * Needs an app, and the account's `events:background` permission for it. On a
+ * website with nobody signed in, opens the sign-in first; rejects
+ * `auth_canceled` if the visitor closes it.
+ *
+ * @this {import('./index.js').EventsModule}
+ * @param {OnScheduleOptions} options
+ * @returns {Promise<Schedule & { off: () => Promise<void> }>} The schedule,
+ *   with `off()` to remove it.
+ */
+export async function onSchedule (options) {
+    const { puter } = this;
+    if ( ! options || typeof options !== 'object' ) {
+        throw new PuterJSError(
+            '`onSchedule()` takes an options object',
+            'invalid_request',
+        );
+    }
+    const { handler, handlerName } = options;
+    if ( handler !== undefined && handler !== null && ! handlerName ) {
+        throw new PuterJSError(
+            'An inline `handler` needs a `handlerName` to check it against',
+            'events_handler_name_required',
+        );
+    }
+    if ( ! handlerName ) {
+        throw new PuterJSError(
+            'A schedule needs a `handlerName`',
+            'events_handler_required',
+        );
+    }
+    for ( const key of ['at', 'startAt', 'expiresAt'] ) {
+        const value = options[key];
+        if ( value instanceof Date && ! Number.isFinite(value.getTime()) ) {
+            throw new PuterJSError(
+                `\`${key}\` is not a valid date`,
+                'events_schedule_invalid',
+            );
+        }
+    }
+
+    const inline = handler === undefined || handler === null
+        ? null
+        : await prepareHandler(puter, handler);
+    const handlerHash = inline?.hash ?? options.handlerHash;
+
+    /** @type {Record<string, unknown>} */
+    const body = { handlerName };
+    for ( const key of TIMING_OPTIONS ) {
+        if ( options[key] !== undefined && options[key] !== null )
+            body[key] = options[key];
+    }
+    if ( handlerHash !== undefined && handlerHash !== null )
+        body.handlerHash = handlerHash;
+    // Serialized only to check it against the cap before the round trip.
+    if ( serializeContext(options.context) !== undefined )
+        body.context = options.context;
+
+    const schedule = /** @type {Schedule} */ (
+        await request(puter, '/events/schedules', body)
+    );
+    const off = async () => this.removeSchedule(schedule.scheduleId);
+    return Object.assign(schedule, { off });
+}
+
+/**
+ * Lists the schedules this caller holds, one page at a time. Ended schedules
+ * stay listed for 7 days with why they ended.
+ *
+ * An app sees only the schedules it created. A session acting for the account
+ * sees them all, including ones left behind by an app it no longer uses.
+ * `context` values are never returned; a schedule reports its key names and a
+ * hash instead.
+ *
+ * On a website with nobody signed in, opens the sign-in first; rejects
+ * `auth_canceled` if the visitor closes it.
+ *
+ * @this {import('./index.js').EventsModule}
+ * @param {ScheduleListOptions} [options]
+ * @returns {Promise<SchedulePage>}
+ */
+export async function listSchedules (options = {}) {
+    const { limit, cursor, includeTotal } = options ?? {};
+    if ( includeTotal !== undefined && typeof includeTotal !== 'boolean' ) {
+        throw new PuterJSError(
+            '`includeTotal` must be a boolean',
+            'invalid_request',
+        );
+    }
+    return /** @type {SchedulePage} */ (
+        /** @type {unknown} */ (
+            await request(this.puter, '/events/schedules', undefined, {
+                ...(limit !== undefined ? { limit } : {}),
+                ...(cursor ? { cursor } : {}),
+                ...(includeTotal ? { includeTotal: true } : {}),
+            })
+        )
+    );
+}
+
+/**
+ * Reads one schedule, ended or not.
+ *
+ * An id this caller does not hold reads as absent, the same as one that never
+ * existed: `events_schedule_not_found`.
+ *
+ * @this {import('./index.js').EventsModule}
+ * @param {string} scheduleId The `scheduleId` of the schedule to read.
+ * @returns {Promise<Schedule>}
+ */
+export async function getSchedule (scheduleId) {
+    assertScheduleId(scheduleId);
+    return /** @type {Schedule} */ (
+        await request(
+            this.puter,
+            `/events/schedules/${encodeURIComponent(scheduleId)}`,
+        )
+    );
+}
+
+/**
+ * Ends a schedule.
+ *
+ * An id this caller does not hold, or one already ended, reads as absent:
+ * `events_schedule_not_found`.
+ *
+ * @this {import('./index.js').EventsModule}
+ * @param {string} scheduleId The `scheduleId` of the schedule to end.
+ * @returns {Promise<void>}
+ */
+export async function removeSchedule (scheduleId) {
+    assertScheduleId(scheduleId);
+    await request(this.puter, '/events/schedules/remove', { scheduleId });
+}

src/puter-js/src/modules/events/schedules.test.js

@@ -0,0 +1,245 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { PuterJSError } from '../../lib/PuterJSError.js';
+
+// Every schedule verb goes through the one HTTP helper, so mocking it needs no
+// server and still exercises the real request bodies.
+const mockRequest = vi.fn();
+vi.mock('./lib/api.js', () => ({
+    request: (...args) => mockRequest(...args),
+}));
+
+const { getSchedule, listSchedules, onSchedule, removeSchedule } =
+    await import('./schedules.js');
+
+const HANDLER = ({ event, ctx }) => fetch(ctx.url, { body: event.id });
+const SCHEDULE_ID = '2f7c1b6e-4d1a-4c55-9a8e-0e6f3c2b1a90';
+
+const makeModule = () => {
+    const module = {
+        puter: { APIOrigin: 'https://api.test', fs: { read: vi.fn() } },
+        onSchedule,
+        listSchedules,
+        getSchedule,
+        removeSchedule,
+    };
+    for ( const name of ['onSchedule', 'listSchedules', 'getSchedule', 'removeSchedule'] )
+        module[name] = module[name].bind(module);
+    return module;
+};
+
+const routeOf = (index = 0) => mockRequest.mock.calls[index][1];
+const bodyOf = (index = 0) => mockRequest.mock.calls[index][2];
+const queryOf = (index = 0) => mockRequest.mock.calls[index][3];
+
+const rejects = async (run) => {
+    try {
+        await run();
+    } catch ( error ) {
+        return error;
+    }
+    throw new Error('expected a rejection');
+};
+
+beforeEach(() => {
+    mockRequest.mockReset();
+    mockRequest.mockResolvedValue({});
+});
+
+describe('onSchedule', () => {
+    it('sends the timing options and the handler name, and returns the server`s schedule', async () => {
+        const view = { scheduleId: SCHEDULE_ID, kind: 'recurring', nextRunAt: 1 };
+        mockRequest.mockResolvedValue(view);
+
+        const schedule = await makeModule().onSchedule({
+            cron: '0 9 * * 1-5',
+            tz: 'America/New_York',
+            startAt: '2026-11-02',
+            expiresAt: 1798761600000,
+            maxRuns: 10,
+            handlerName: 'digest',
+            unknownOption: true,
+        });
+
+        expect(routeOf()).toBe('/events/schedules');
+        expect(bodyOf()).toEqual({
+            cron: '0 9 * * 1-5',
+            tz: 'America/New_York',
+            startAt: '2026-11-02',
+            expiresAt: 1798761600000,
+            maxRuns: 10,
+            handlerName: 'digest',
+        });
+        expect(schedule).toBe(view);
+        expect(schedule.nextRunAt).toBe(1);
+    });
+
+    it('passes `every`, `at` and a `Date` through for the server to check', async () => {
+        const at = new Date('2026-11-01T18:30:00Z');
+        await makeModule().onSchedule({ every: '6h', handlerName: 'poll' });
+        await makeModule().onSchedule({ at, handlerName: 'remind' });
+
+        expect(bodyOf(0)).toEqual({ every: '6h', handlerName: 'poll' });
+        expect(bodyOf(1)).toEqual({ at, handlerName: 'remind' });
+        expect(JSON.parse(JSON.stringify(bodyOf(1))).at).toBe(
+            '2026-11-01T18:30:00.000Z',
+        );
+    });
+
+    it('rejects an invalid Date for `at`, `startAt` or `expiresAt` before any request', async () => {
+        for ( const key of ['at', 'startAt', 'expiresAt'] ) {
+            const error = await rejects(() =>
+                makeModule().onSchedule({
+                    ...(key === 'at' ? {} : { every: '1h' }),
+                    [key]: new Date(NaN),
+                    handlerName: 'tick',
+                    handler: HANDLER,
+                }));
+            expect(error).toBeInstanceOf(PuterJSError);
+            expect(error.code).toBe('events_schedule_invalid');
+            expect(error.message).toContain(key);
+        }
+        expect(mockRequest).not.toHaveBeenCalled();
+    });
+
+    it('hands back a handle whose off() removes the schedule', async () => {
+        const module = makeModule();
+        mockRequest.mockResolvedValueOnce({ scheduleId: SCHEDULE_ID });
+        const schedule = await module.onSchedule({ every: '1h', handlerName: 'poll' });
+
+        await schedule.off();
+
+        expect(routeOf(1)).toBe('/events/schedules/remove');
+        expect(bodyOf(1)).toEqual({ scheduleId: SCHEDULE_ID });
+    });
+
+    it('sends an inline handler as a hash, never as source', async () => {
+        const { hashSource } = await import('./lib/handlerSource.js');
+        await makeModule().onSchedule({
+            every: '1h',
+            handlerName: 'poll',
+            handler: HANDLER,
+        });
+
+        expect(bodyOf().handlerHash).toBe(
+            await hashSource(Function.prototype.toString.call(HANDLER)),
+        );
+        expect(JSON.stringify(bodyOf())).not.toContain('fetch(');
+    });
+
+    it('passes a handlerHash given in place of the handler', async () => {
+        const handlerHash = 'a'.repeat(64);
+        await makeModule().onSchedule({ every: '1h', handlerName: 'poll', handlerHash });
+        expect(bodyOf().handlerHash).toBe(handlerHash);
+    });
+
+    it('forwards any handlerHash it is given, an empty one included, for the server to judge', async () => {
+        await makeModule().onSchedule({ every: '1h', handlerName: 'poll', handlerHash: '' });
+        await makeModule().onSchedule({ every: '1h', handlerName: 'poll', handlerHash: null });
+
+        expect(bodyOf(0)).toHaveProperty('handlerHash', '');
+        expect(bodyOf(1)).not.toHaveProperty('handlerHash');
+    });
+
+    it('needs a handler name before it calls anything', async () => {
+        const unnamed = await rejects(() => makeModule().onSchedule({ every: '1h' }));
+        expect(unnamed.code).toBe('events_handler_required');
+
+        const inline = await rejects(() =>
+            makeModule().onSchedule({ every: '1h', handler: HANDLER }),
+        );
+        expect(inline.code).toBe('events_handler_name_required');
+
+        const none = await rejects(() => makeModule().onSchedule());
+        expect(none.code).toBe('invalid_request');
+        expect(mockRequest).not.toHaveBeenCalled();
+    });
+
+    it('refuses a context over the cap before the round trip', async () => {
+        const error = await rejects(() =>
+            makeModule().onSchedule({
+                every: '1h',
+                handlerName: 'poll',
+                context: { blob: 'x'.repeat(5000) },
+            }),
+        );
+        expect(error.code).toBe('events_context_too_large');
+        expect(mockRequest).not.toHaveBeenCalled();
+    });
+
+    it('sends the context it was given', async () => {
+        await makeModule().onSchedule({
+            every: '1h',
+            handlerName: 'poll',
+            context: { team: 'design' },
+        });
+        expect(bodyOf().context).toEqual({ team: 'design' });
+    });
+
+    it('passes a server refusal through unchanged, a disabled server included', async () => {
+        for ( const code of ['events_disabled', 'events_schedule_invalid', 'events_schedule_busy'] ) {
+            const refusal = new PuterJSError('refused', code);
+            mockRequest.mockRejectedValueOnce(refusal);
+            const error = await rejects(() =>
+                makeModule().onSchedule({ every: '1h', handlerName: 'poll' }),
+            );
+            expect(error).toBe(refusal);
+        }
+    });
+});
+
+describe('listSchedules', () => {
+    it('asks for one page, with only the paging options given', async () => {
+        const page = { items: [], cursor: 'next' };
+        mockRequest.mockResolvedValue(page);
+
+        const result = await makeModule().listSchedules({
+            limit: 2,
+            cursor: 'abc',
+            includeTotal: true,
+        });
+        await makeModule().listSchedules();
+        await makeModule().listSchedules({ cursor: null, includeTotal: false });
+
+        expect(result).toBe(page);
+        expect(routeOf(0)).toBe('/events/schedules');
+        expect(bodyOf(0)).toBeUndefined();
+        expect(queryOf(0)).toEqual({ limit: 2, cursor: 'abc', includeTotal: true });
+        expect(queryOf(1)).toEqual({});
+        expect(queryOf(2)).toEqual({});
+    });
+
+    it('refuses an includeTotal that is not a boolean', async () => {
+        const error = await rejects(() =>
+            makeModule().listSchedules({ includeTotal: 'yes' }),
+        );
+        expect(error.code).toBe('invalid_request');
+        expect(mockRequest).not.toHaveBeenCalled();
+    });
+});
+
+describe('getSchedule and removeSchedule', () => {
+    it('name the schedule in the path and the body', async () => {
+        await makeModule().getSchedule(SCHEDULE_ID);
+        await makeModule().removeSchedule(SCHEDULE_ID);
+
+        expect(routeOf(0)).toBe(`/events/schedules/${SCHEDULE_ID}`);
+        expect(bodyOf(0)).toBeUndefined();
+        expect(routeOf(1)).toBe('/events/schedules/remove');
+        expect(bodyOf(1)).toEqual({ scheduleId: SCHEDULE_ID });
+    });
+
+    it('keep an id inside its path segment', async () => {
+        await makeModule().getSchedule('../subscriptions');
+        expect(routeOf()).toBe('/events/schedules/..%2Fsubscriptions');
+    });
+
+    it('answer an id that is not one as absent, without a round trip', async () => {
+        for ( const id of [undefined, null, '', '  ', 42] ) {
+            for ( const verb of ['getSchedule', 'removeSchedule'] ) {
+                const error = await rejects(() => makeModule()[verb](id));
+                expect(error.code).toBe('events_schedule_not_found');
+            }
+        }
+        expect(mockRequest).not.toHaveBeenCalled();
+    });
+});

src/puter-js/src/modules/events/types.js

@@ -286,6 +286,160 @@
  *   `onPersistent()` returns, not on one a listing reports.
  */
 
+/**
+ * What a handler a schedule runs gets as `event`. Runs are at least once: a
+ * retry, or a run that outlives its timeout, can run one slot twice, with the
+ * same `id`. Skip work already done for an `id`.
+ *
+ * @typedef {Object} PuterScheduleEvent
+ * @property {string} subject `schedule:<scheduleId>`.
+ * @property {'tick'} op Always `'tick'`.
+ * @property {string} id `<scheduleId>:<scheduledFor>`, the same on every
+ *   attempt at one run.
+ * @property {string} scheduleId The schedule this run belongs to.
+ * @property {number} scheduledFor When this run was due, in milliseconds since
+ *   the epoch.
+ * @property {number} firedAt When this attempt was sent to the handler, in
+ *   milliseconds since the epoch.
+ * @property {number} missed Runs skipped since the last one that ran, such as
+ *   while the schedule was suspended. Only this run happens for them.
+ * @property {true} [missedCapped] Present when `missed` stopped counting at
+ *   its cap of 1,000.
+ * @property {number} attempt Attempts at this run, this one included.
+ */
+
+/**
+ * What a handler a schedule runs is called with, in the app's events worker.
+ *
+ * @typedef {Object} ScheduleDelivery
+ * @property {PuterScheduleEvent} event The run.
+ * @property {Readonly<Record<string, unknown>>} [ctx] The `context` the
+ *   schedule was created with, frozen.
+ * @property {unknown} [user] A `puter` bound to the account holding the
+ *   schedule.
+ * @property {(input: unknown, init?: unknown) => Promise<unknown>} [fetch]
+ *   `puter.net.fetch` where it exists, the environment's `fetch` otherwise.
+ */
+
+/**
+ * A handler a schedule runs. Its return value is ignored.
+ *
+ * @typedef {(delivery: ScheduleDelivery) => unknown} ScheduleHandler
+ */
+
+/**
+ * Options for {@link import('./schedules.js').onSchedule}. Give exactly one
+ * of `every`, `cron` or `at`. Times are an ISO-8601 string with a zone (or a
+ * plain date, read as UTC midnight), a `Date`, or milliseconds since the
+ * epoch, and are rounded up to the next whole second.
+ *
+ * @typedef {Object} OnScheduleOptions
+ * @property {number | string} [every] Run at this interval: milliseconds, or
+ *   a duration like `'30m'`, `'6h'` or `'1d'` (`s`, `m`, `h`, `d`, `w`). A
+ *   whole number of seconds, at least the plan's minimum, at most 366 days.
+ * @property {string} [cron] Run on a 5-field cron expression, or a nickname
+ *   such as `'@daily'`. Standard fields only.
+ * @property {string} [tz] The IANA time zone `cron` runs in. Defaults to
+ *   `'UTC'`. Only with `cron`.
+ * @property {number | string | Date} [at] Run once, at this time. At least 1
+ *   second and at most 366 days ahead.
+ * @property {number | string | Date} [startAt] For `every`, the first run;
+ *   without it, the first run is one interval after create. A past `startAt`
+ *   aligns the runs to it, starting at the next one. For `cron`, the earliest
+ *   run.
+ * @property {number | string | Date} [expiresAt] The last time the schedule may
+ *   run, inclusive.
+ * @property {number} [maxRuns] Stop after this many runs (1 to 100,000). With
+ *   `expiresAt` too, the earlier end wins.
+ * @property {string} handlerName The published handler each run calls.
+ * @property {ScheduleHandler | string | { file: string }} [handler] The handler
+ *   source this schedule was written against. Sent as a hash: the schedule is
+ *   refused unless it matches what is published under `handlerName`. It never
+ *   runs in this client.
+ * @property {string} [handlerHash] The SHA-256 hex digest of that source, in
+ *   place of `handler`.
+ * @property {Record<string, unknown>} [context] Values the handler needs,
+ *   evaluated **now** and delivered as a frozen `ctx` on every run. Capped at
+ *   4 KB serialized, and can't be changed after create.
+ */
+
+/**
+ * A schedule, as its holder sees it. Times are milliseconds since the epoch.
+ * `context` values are never returned, only which keys it sets and a hash.
+ *
+ * @typedef {Object} Schedule
+ * @property {string} scheduleId The schedule's id, and what
+ *   `removeSchedule()` and `getSchedule()` name.
+ * @property {'recurring' | 'once' | 'deferred'} kind `recurring` for `every`
+ *   and `cron`, `once` for `at`, `deferred` for a delayed notification.
+ * @property {string} appUid The app that created it.
+ * @property {string | null} handlerName The handler each run calls, or `null`
+ *   once the schedule has ended.
+ * @property {string[] | null} contextKeys Key names of its context, or `null`
+ *   when it has none or has ended.
+ * @property {string | null} contextHash Hash of its context.
+ * @property {number | null} every The interval of an `every` schedule, in
+ *   milliseconds.
+ * @property {string | null} cron The cron expression of a `cron` schedule.
+ * @property {string} tz The time zone `cron` runs in; `'UTC'` otherwise.
+ * @property {number} startAt The first allowed run. For `once`, its only run.
+ * @property {number | null} endsAt The last allowed run, inclusive, from
+ *   `expiresAt` or `maxRuns`. For `once`, its only run.
+ * @property {'active' | 'suspended' | 'sending' | 'ended'} state `suspended`
+ *   while it doesn't run but can resume, `sending` once a delayed
+ *   notification has started to go (it can't be cancelled then), `ended` once
+ *   it never runs again.
+ * @property {string | null} suspendedReason Why it is suspended:
+ *   `handler_not_found`, `failures` or `no_credit`.
+ * @property {string | null} endReason Why it ended: `removed`, `ended` (its
+ *   last run is done), `app_deleted`, `uninstalled`, `permission_revoked`,
+ *   `holder_gone`, `app_gone`, `orphaned`, `delivered` or `dropped`.
+ * @property {number | null} endedAt When it ended.
+ * @property {number | null} nextRunAt The next slot of its definition, or
+ *   `null` while suspended or sending, and once ended. A run starts within about a minute
+ *   of it; a late or retried run for an earlier slot may fire first.
+ * @property {number} createdAt When it was created.
+ * @property {() => Promise<void>} [off] Removes the schedule. Present on the
+ *   schedule `onSchedule()` returns, not on one a listing reports.
+ */
+
+/**
+ * Options for {@link import('./schedules.js').listSchedules}.
+ *
+ * @typedef {Object} ScheduleListOptions
+ * @property {number} [limit] Schedules per page. Defaults to 50, capped at 200.
+ * @property {string | null} [cursor] The `cursor` from the previous page.
+ *   Absent or `null` starts from the first page.
+ * @property {boolean} [includeTotal] Adds `total`, the count across every page.
+ */
+
+/**
+ * Error codes the schedule methods reject with, besides the shared events
+ * codes such as `events_disabled` and `too_many_requests`.
+ *
+ * @typedef {'bad_request'
+ *   | 'invalid_request'
+ *   | 'events_schedule_invalid'
+ *   | 'events_schedule_never_fires'
+ *   | 'events_schedule_too_frequent'
+ *   | 'events_schedule_app_required'
+ *   | 'events_schedule_limit'
+ *   | 'events_schedule_budget'
+ *   | 'events_schedule_not_found'
+ *   | 'events_schedule_sending'
+ *   | 'events_schedule_busy'
+ *   | 'events_handler_required'
+ *   | 'events_handler_name_required'
+ *   | 'events_handler_invalid'
+ *   | 'events_handler_hash_unavailable'
+ *   | 'events_handler_not_found'
+ *   | 'events_handler_hash_mismatch'
+ *   | 'events_context_invalid'
+ *   | 'events_context_too_large'
+ *   | 'events_durable_requires_account'
+ *   | 'events_background_consent_required'} ScheduleErrorCode
+ */
+
 /**
  * Where a handler operation applies. An app token publishes into its own app
  * and needs neither field; an account session has to name an app it owns.