Compare commits

..
Author SHA1 Message Date
ProgrammerIn-wonderlandandClaude Opus 5.5 2f35dadb28 fix(hosting): choose the site and subdomain from Host, not X-Forwarded-Host
With trust proxy on, express derives req.hostname and req.subdomains from
X-Forwarded-Host, which any client can send, while host validation reads the
raw Host. A request could serve one hosted site's files on the main domain,
or reach the API on a hosting host. The header is now dropped before routing.

Fixes PUT-2272

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-11-02 18:32:03 -05:00
274 changed files with 10100 additions and 26277 deletions

No files matched your search

-48
View File
@@ -1,48 +0,0 @@
name: Notify HeyPuter (dev)
on:
workflow_dispatch:
inputs:
puter_ref:
description: "puter branch, tag or SHA"
type: string
default: "main"
permissions:
contents: read
jobs:
authorize:
if: github.repository == 'HeyPuter/puter'
runs-on: ubuntu-latest
steps:
- name: Require maintain or admin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ACTOR: ${{ github.triggering_actor }}
run: |
ROLE=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$ACTOR/permission" --jq .role_name)
if [ "$ROLE" != "maintain" ] && [ "$ROLE" != "admin" ]; then
echo "::error::$ACTOR has role '$ROLE'; maintain or admin required"
exit 1
fi
notify:
needs: authorize
runs-on: ubuntu-latest
# Holds HEYPUTER_DISPATCH_TOKEN; limited to main so branch copies of this file can't read it.
environment: heyputer-dispatch
steps:
- name: Trigger heyputer dev deploy
env:
PUTER_REF: ${{ inputs.puter_ref }}
TOKEN: ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}
run: |
jq -nc --arg ref "$PUTER_REF" \
'{event_type: "puter-dev-requested", client_payload: {puter_ref: $ref}}' |
curl -fsS -X POST \
-H "Authorization: token $TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-d @-
echo "Requested \`$PUTER_REF\`" >> "$GITHUB_STEP_SUMMARY"
+1 -2
View File
@@ -8,7 +8,6 @@ on:
jobs:
notify:
runs-on: ubuntu-latest
environment: heyputer-dispatch
steps:
- name: Trigger heyputer build
run: |
@@ -16,4 +15,4 @@ jobs:
-H "Authorization: token ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
-d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
-3
View File
@@ -63,9 +63,6 @@ store:
sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
among rows up to that id. Each is its own indexed read, merged into one
page, so a row replaced after a client read it comes back from that cursor.
`from=end` with no cursor answers an empty page whose cursor holds the
slice's newest `id` and the slice's latest `(replaced_at, id)` as the
replacement mark.
Cursors that carry a sort also pin it: a request that passes a cursor plus a
conflicting sort is rejected with 400.
+10 -5
View File
@@ -257,6 +257,8 @@ describe('whoami extension — handleWhoami', () => {
expect(body.created_ts).toBeGreaterThan(0);
// `directories` is only sent to user actors — confirm it’s present.
expect(body.directories).toBeDefined();
// taskbar_items is only sent to user actors.
expect(body).toHaveProperty('taskbar_items');
});
it('reports the SMS-to-card fallback only once a send has opened it', async () => {
@@ -320,7 +322,7 @@ describe('whoami extension — handleWhoami', () => {
expect(flags.payment_bypass).toBeUndefined();
});
it('strips created_ts and human_readable_age fields for app actors', async () => {
it('strips desktop_bg_*, created_ts and human_readable_age fields for app actors', async () => {
const user = await seedUser();
const { res, captured } = makeRes();
@@ -338,6 +340,9 @@ describe('whoami extension — handleWhoami', () => {
expect(body.app_name).toBe('app-test-actor');
// is_user_token is stripped for app actors.
expect(body.is_user_token).toBeUndefined();
expect(body.desktop_bg_url).toBeUndefined();
expect(body.desktop_bg_color).toBeUndefined();
expect(body.desktop_bg_fit).toBeUndefined();
expect(body.human_readable_age).toBeUndefined();
// Account age, in either form, is not exposed to apps.
expect(body.created_ts).toBeUndefined();
@@ -427,7 +432,7 @@ describe('whoami extension — handleWhoami', () => {
});
});
it('cannot add directories for an app actor, which core omits', async () => {
it('cannot add taskbar_items for an app actor, which core omits', async () => {
const user = await seedUser();
const body = await whoamiWithListener(
makeActor({
@@ -435,11 +440,11 @@ describe('whoami extension — handleWhoami', () => {
app: { uid: 'app-test-actor' },
}),
(_key, event) => {
event.details.directories = { '/injected': 'x' };
event.details.taskbar_items = [{ name: 'injected' }];
},
);
expect(body).not.toHaveProperty('directories');
expect(body).not.toHaveProperty('taskbar_items');
});
});
@@ -520,7 +525,7 @@ describe('whoami extension — handleWhoami', () => {
expect(body.otp_secret).toBeUndefined();
expect(body.signup_ip).toBeUndefined();
// Not just absent as a key — the values must not appear anywhere
// in the payload (nested under metadata, …).
// in the payload (nested under metadata, taskbar items, …).
const serialized = JSON.stringify(body);
expect(serialized).not.toContain('5551234567');
expect(serialized).not.toContain('fp_ABC123');
+17 -3
View File
@@ -1,5 +1,6 @@
import { Context } from '@heyputer/backend/src/core';
import { extension } from '@heyputer/backend/src/extensions';
import { APP_ICON_SIZES } from '@heyputer/backend/src/util/appIcon.js';
import {
buildUserDetails,
scrubSensitive,
@@ -25,7 +26,7 @@ const APP_VISIBLE_LISTENER_KEYS: ReadonlySet<string> = new Set([
]);
export const handleWhoami = async (
_req: Request,
req: Request,
res: Response,
): Promise<void> => {
const actor = Context.get('actor');
@@ -53,10 +54,19 @@ export const handleWhoami = async (
return;
}
const rawIconSize =
typeof req.query?.icon_size === 'string'
? Number(req.query.icon_size)
: undefined;
const iconSize =
rawIconSize !== undefined && APP_ICON_SIZES.includes(rawIconSize)
? rawIconSize
: undefined;
const details = await buildUserDetails(
user,
{ config: extension.config, clients, stores, services },
{ isUser },
{ isUser, iconSize, noIcons: !iconSize },
);
details.human_readable_age = user.timestamp
? timeago.format(new Date(user.timestamp as string))
@@ -71,6 +81,9 @@ export const handleWhoami = async (
delete details.email;
delete details.unconfirmed_email;
}
delete details.desktop_bg_url;
delete details.desktop_bg_color;
delete details.desktop_bg_fit;
delete details.human_readable_age;
delete details.created_ts;
delete details.is_user_token;
@@ -108,7 +121,8 @@ export const handleWhoami = async (
}
const subscription = details.subscription as
{ offering?: Record<string, unknown> } | undefined;
| { offering?: Record<string, unknown> }
| undefined;
if (subscription?.offering) {
delete subscription.offering.group;
delete subscription.offering.benefits;
@@ -285,18 +285,6 @@ describe('AbstractDatabaseClient — SQL generation', () => {
);
});
it('reads the database clock as epoch milliseconds in each dialect', () => {
expect(sqlite().nowMsExpr()).toBe(
"CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
);
expect(mysql().nowMsExpr()).toBe(
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
);
expect(postgres().nowMsExpr()).toBe(
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
);
});
it('describes a stored timestamp the way each driver reads it back', () => {
const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');
@@ -401,21 +401,6 @@ export class AbstractDatabaseClient extends PuterClient {
});
}
/**
* The database's own clock as epoch milliseconds, so nodes whose clocks
* disagree still agree on it. The mysql form ignores the session time zone,
* so a DST change can't shift it.
*/
nowMsExpr(): string {
return this.case({
sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
postgres:
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
otherwise:
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
});
}
/**
* What a read of a timestamp column holding `seconds` returns: sqlite gives
* back the UTC text it stored, mysql2 and pg a `Date` read in node's time
@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
/** Highest schema version the migration table can reach. */
const CURRENT_SCHEMA_VERSION = 98;
const CURRENT_SCHEMA_VERSION = 95;
/**
* These suites migrate real files on disk. Idle they finish in well under a
@@ -724,85 +724,6 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
]);
});
it('keeps notification.send_key unique', async () => {
const rows = await client.read('PRAGMA index_list(`notification`)');
expect(
rows.find((r) => r.name === 'idx_notification_send_key'),
).toMatchObject({ unique: 1 });
const columns = await client.read(
'PRAGMA index_info(`idx_notification_send_key`)',
);
expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
});
it('creates event_user_handlers with its unique and holder indexes', async () => {
const columns = await client.read(
'PRAGMA table_info(`event_user_handlers`)',
);
expect(columns.map((c) => String(c.name))).toEqual([
'id',
'app_uid',
'holder_user_id',
'name',
'source',
'source_hash',
'created_at',
'updated_at',
'scope_since',
]);
const indexes = await client.read(
'PRAGMA index_list(`event_user_handlers`)',
);
expect(
indexes.find(
(i) => i.name === 'idx_event_user_handlers_app_holder_name',
),
).toMatchObject({ unique: 1 });
expect(
indexes.find((i) => i.name === 'idx_event_user_handlers_holder_app'),
).toMatchObject({ unique: 0 });
const columnsOf = async (index: string) =>
(await client.read(`PRAGMA index_info(\`${index}\`)`)).map((c) =>
String(c.name),
);
expect(await columnsOf('idx_event_user_handlers_app_holder_name')).toEqual(
['app_uid', 'holder_user_id', 'name'],
);
expect(await columnsOf('idx_event_user_handlers_holder_app')).toEqual([
'holder_user_id',
'app_uid',
]);
});
it('leaves event_handlers as it was', async () => {
const columns = await client.read('PRAGMA table_info(`event_handlers`)');
expect(columns.map((c) => String(c.name))).toEqual([
'id',
'app_uid',
'name',
'source',
'source_hash',
'created_at',
'updated_at',
]);
});
it('adds event_subscriptions.handler_scope, app scope for existing rows', async () => {
const columns = await client.read(
'PRAGMA table_info(`event_subscriptions`)',
);
expect(columns.find((c) => c.name === 'handler_scope')).toMatchObject({
notnull: 1,
dflt_value: '0',
});
const indexes = await client.read(
'PRAGMA index_list(`event_subscriptions`)',
);
expect(indexes.map((i) => String(i.name))).toContain(
'idx_event_subscriptions_holder',
);
});
it('leaves an already-migrated database untouched on a second boot', async () => {
const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
const path = join(dir, 'nested', 'puter.sqlite');
@@ -134,9 +134,6 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
[92, ['0097_user-to-user-holder-permission-index.sql']],
[93, ['0098_notification-groups.sql']],
[94, ['0099_kv-share-handle-notify.sql']],
[95, ['0100_notification-send-key.sql']],
[96, ['0101_user-scoped-handlers.sql']],
[97, ['0102_notification-scope-id-index.sql']],
];
export class SqliteDatabaseClient extends AbstractDatabaseClient {
@@ -1,43 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
-- rationale. The key is a uuid, so ascii and compared byte for byte. No
-- per-file applied-state tracking, so every step is guarded and a replay does
-- nothing.
CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
DELIMITER //
CREATE PROCEDURE _puter_add_notification_send_key_index()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'notification'
AND INDEX_NAME = 'idx_notification_send_key'
) THEN
ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
(`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
END IF;
END//
DELIMITER ;
CALL _puter_add_notification_send_key_index();
DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
@@ -1,66 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Handlers an app's user published into their own scope, kept apart from
-- `event_handlers` so code that predates them never reads them. See
-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
--
-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline,
-- and a guarded column add. There is no per-file applied-state tracking, so a
-- replay has to be a no-op.
CREATE TABLE IF NOT EXISTS `event_user_handlers` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
-- Matches `apps`.`uid` exactly, charset included, so an equality against
-- one never falls back to a conversion.
`app_uid` char(40) CHARACTER SET latin1 COLLATE latin1_swedish_ci NOT NULL,
`holder_user_id` int unsigned NOT NULL,
`name` varchar(128) NOT NULL,
`source` mediumtext NOT NULL,
`source_hash` char(64) NOT NULL,
`created_at` bigint NOT NULL,
`updated_at` bigint NOT NULL,
-- When the scope first held a handler; kept across handler churn, so the
-- rent sweep sees the scope's age, not its oldest remaining handler's.
`scope_since` bigint NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_event_user_handlers_app_holder_name` (`app_uid`, `holder_user_id`, `name`),
KEY `idx_event_user_handlers_holder_app` (`holder_user_id`, `app_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- Guarded here rather than through `_puter_add_col`, so the file also applies
-- on its own to a database that never ran the earlier migrations.
DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;
DELIMITER //
CREATE PROCEDURE _puter_add_event_subscriptions_handler_scope()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'event_subscriptions'
AND COLUMN_NAME = 'handler_scope'
) THEN
ALTER TABLE `event_subscriptions`
ADD COLUMN `handler_scope` tinyint unsigned NOT NULL DEFAULT 0,
ALGORITHM=INSTANT;
END IF;
END//
DELIMITER ;
CALL _puter_add_event_subscriptions_handler_scope();
DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;
@@ -1,41 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification` (`user_id`, `audience`, `app_uid`, `id`). See
-- sqlite/0102_notification-scope-id-index.sql for the rationale. No per-file
-- applied-state tracking, so the index is added only when missing and a
-- replay does nothing.
DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;
DELIMITER //
CREATE PROCEDURE _puter_add_notification_scope_id_index()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'notification'
AND INDEX_NAME = 'idx_notification_scope_id'
) THEN
ALTER TABLE `notification` ADD INDEX `idx_notification_scope_id`
(`user_id`, `audience`, `app_uid`, `id`), ALGORITHM=INPLACE, LOCK=NONE;
END IF;
END//
DELIMITER ;
CALL _puter_add_notification_scope_id_index();
DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;
@@ -1,24 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
-- rationale. Idempotent via IF NOT EXISTS.
ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
ON notification (send_key);
@@ -1,39 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Handlers an app's user published into their own scope. See
-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
-- Idempotent via IF NOT EXISTS.
CREATE TABLE IF NOT EXISTS event_user_handlers (
id BIGSERIAL PRIMARY KEY,
app_uid VARCHAR(40) NOT NULL,
holder_user_id INTEGER NOT NULL,
name VARCHAR(128) NOT NULL,
source TEXT NOT NULL,
source_hash VARCHAR(64) NOT NULL,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
scope_since BIGINT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_event_user_handlers_app_holder_name
ON event_user_handlers (app_uid, holder_user_id, name);
CREATE INDEX IF NOT EXISTS idx_event_user_handlers_holder_app
ON event_user_handlers (holder_user_id, app_uid);
ALTER TABLE event_subscriptions ADD COLUMN IF NOT EXISTS handler_scope smallint NOT NULL DEFAULT 0;
@@ -1,23 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- notification (user_id, audience, app_uid, id). See
-- sqlite/0102_notification-scope-id-index.sql for the rationale. Idempotent via
-- IF NOT EXISTS.
CREATE INDEX IF NOT EXISTS idx_notification_scope_id
ON notification (user_id, audience, app_uid, id);
@@ -1,27 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification`.`send_key`: the key of the delayed send (a `deferred`
-- schedule) that wrote this row or last replaced it, derived from the
-- schedule and its slot. Unique, so one slot can never write two rows, and a
-- send that crashed after writing finds its row again by it. Null on every row
-- a delayed send never touched.
ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
ON `notification` (`send_key`);
@@ -1,48 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Handlers an app's user published into their own scope, in a table of their
-- own so `event_handlers` stays the app owner's set. Same columns as
-- `event_handlers` (see 0076_event-handlers.sql) plus the holder.
--
-- - `holder_user_id` : the user whose scope this is. A name is unique per
-- (app, holder), so the same name can live in the app's
-- set and in each holder's.
-- - `(holder_user_id, app_uid)` : a holder's scopes, for listing and for the
-- rent count.
--
-- `event_subscriptions.handler_scope` says which table a row's `handler_name`
-- refers to: 0 the app's set, 1 the row holder's scope.
CREATE TABLE IF NOT EXISTS `event_user_handlers` (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"app_uid" TEXT NOT NULL,
"holder_user_id" INTEGER NOT NULL,
"name" TEXT NOT NULL,
"source" TEXT NOT NULL,
"source_hash" TEXT NOT NULL,
"created_at" INTEGER NOT NULL, -- unix seconds
"updated_at" INTEGER NOT NULL,
"scope_since" INTEGER NOT NULL -- unix seconds; the scope's first handler, kept across churn
);
CREATE UNIQUE INDEX IF NOT EXISTS `idx_event_user_handlers_app_holder_name`
ON `event_user_handlers` (`app_uid`, `holder_user_id`, `name`);
CREATE INDEX IF NOT EXISTS `idx_event_user_handlers_holder_app`
ON `event_user_handlers` (`holder_user_id`, `app_uid`);
ALTER TABLE `event_subscriptions` ADD COLUMN `handler_scope` INTEGER NOT NULL DEFAULT 0;
@@ -1,22 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Index a mailbox slice by row id, so a fetch finds the newest row of a slice,
-- and pages forward through one, with a seek rather than a scan of the slice.
CREATE INDEX IF NOT EXISTS `idx_notification_scope_id`
ON `notification` (`user_id`, `audience`, `app_uid`, `id`);
+12 -19
View File
@@ -581,8 +581,6 @@ export type EventMap = {
updated?: true;
/** Names this one send, for a receipt asked about it. */
sendId: string;
/** The same `sendId` may be emitted again, as a delayed send is. */
replayable?: true;
};
/**
* One notification, addressed at whatever sockets each region holds for the
@@ -666,18 +664,9 @@ export type EventMap = {
// and going away (1→0, whether by removing the last handler or by the
// destroy route). `actor.user` is the app's owner, not the caller — a
// developer session publishing for an app it owns is the common case, but
// billing follows ownership. For `scope: 'user'` the unit is one user's
// handlers in one app instead, and `actor.user` is that user.
'events.worker.create': {
actor: Actor;
appUid: string;
scope: 'app' | 'user';
};
'events.worker.destroy': {
actor: Actor;
appUid: string;
scope: 'app' | 'user';
};
// billing follows ownership.
'events.worker.create': { actor: Actor; appUid: string };
'events.worker.destroy': { actor: Actor; appUid: string };
// ---- Outer / GUI broadcast ----
'outer.cacheUpdate': {
@@ -1041,10 +1030,11 @@ export type EventKey = keyof EventMap & string;
// Generates a wildcard for every non-final dot-separated prefix of K.
export type WildcardPrefixes<K extends string> =
K extends `${infer Head}.${infer Tail}`
? | `${Head}.*`
| (Tail extends `${string}.${string}`
? `${Head}.${WildcardPrefixes<Tail>}`
: never)
?
| `${Head}.*`
| (Tail extends `${string}.${string}`
? `${Head}.${WildcardPrefixes<Tail>}`
: never)
: never;
export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
@@ -1055,7 +1045,10 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
/** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
export type NotifDeliveryOutcome =
'received' | 'unreceived' | 'no-socket' | 'remote-only';
| 'received'
| 'unreceived'
| 'no-socket'
| 'remote-only';
/** One socket's receipt for a live notification. */
export interface NotifDeliveryResponder {
@@ -23,25 +23,18 @@
*/
import type { fetch as undiciFetch } from 'undici';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { describe, expect, it, vi } from 'vitest';
import type { IConfig } from '../../types.js';
import {
DispatcherInvokeTransport,
EVENTS_DEPLOYED_HEADER,
EVENTS_DEADLINE_MARGIN_MS,
EVENTS_DISPATCH_BATCH_PATH,
EVENTS_DISPATCH_ERROR_HEADER,
EVENTS_DISPATCH_PATH,
EVENTS_ERROR_HEADER,
EVENTS_HANDLED_HEADER,
EVENTS_HOLDER_HEADER,
EVENTS_INVOKE_TIMEOUT_MS,
EVENTS_WORKER_LEASE_FACTOR,
EventsWorkerInvokerClient,
INVOKE_CONNECTIONS,
SCHEDULE_INVOKE_CONNECTIONS,
outcomeForStatus,
parseAnswerLine,
type WorkerInvokeRequest,
} from './EventsWorkerInvokerClient.js';
@@ -80,7 +73,10 @@ const refusedFetch: FetchImpl = (async () => {
}) as unknown as FetchImpl;
/** A stub fetch that records the URL it was called with. */
const capturingFetch = (calls: string[], status = 200): FetchImpl =>
const capturingFetch = (
calls: string[],
status = 200,
): FetchImpl =>
(async (url: string) => {
calls.push(url);
return new Response(null, { status });
@@ -88,13 +84,9 @@ const capturingFetch = (calls: string[], status = 200): FetchImpl =>
describe('DispatcherInvokeTransport', () => {
it('reports a handled 400 with no error — the handler`s own refusal', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: stubFetch(400, { [EVENTS_HANDLED_HEADER]: '1' }),
});
expect(await transport.send(CALL)).toEqual({
status: 400,
handled: true,
@@ -102,13 +94,9 @@ describe('DispatcherInvokeTransport', () => {
});
it('reports an unmarked 404 with no dispatch-error header as unhandled', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: stubFetch(404),
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: stubFetch(404),
});
expect(await transport.send(CALL)).toEqual({
status: 404,
handled: false,
@@ -116,15 +104,9 @@ describe('DispatcherInvokeTransport', () => {
});
it('turns a dispatch-error header into a null status with the reason', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: stubFetch(502, {
[EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed',
}),
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: stubFetch(502, { [EVENTS_DISPATCH_ERROR_HEADER]: 'deploy-failed' }),
});
expect(await transport.send(CALL)).toEqual({
status: null,
error: 'dispatcher: deploy-failed (502)',
@@ -138,13 +120,9 @@ describe('DispatcherInvokeTransport', () => {
headers.push(init?.headers as Record<string, string>);
return new Response(null, { status: 200 });
}) as unknown as FetchImpl;
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl,
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl,
});
await transport.send(CALL);
await transport.send({ ...CALL, deployed: true });
@@ -154,13 +132,9 @@ describe('DispatcherInvokeTransport', () => {
});
it('reports a handled 200 as settled with no error', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: stubFetch(200, { [EVENTS_HANDLED_HEADER]: '1' }),
});
expect(await transport.send(CALL)).toEqual({
status: 200,
handled: true,
@@ -168,13 +142,9 @@ describe('DispatcherInvokeTransport', () => {
});
it('passes a 429 through as-is', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: stubFetch(429),
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: stubFetch(429),
});
expect(await transport.send(CALL)).toEqual({
status: 429,
handled: false,
@@ -182,26 +152,18 @@ describe('DispatcherInvokeTransport', () => {
});
it('answers null and says so when the request times out', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: hangingFetch,
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: hangingFetch,
});
const result = await transport.send({ ...CALL, timeoutMs: 10 });
expect(result.status).toBeNull();
expect(result.timedOut).toBe(true);
});
it('answers null without a timeout when the request fails outright', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: refusedFetch,
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: refusedFetch,
});
expect(await transport.send(CALL)).toEqual({
status: null,
error: 'fetch failed',
@@ -225,16 +187,12 @@ describe('DispatcherInvokeTransport', () => {
});
it('surfaces the runtime`s own error header alongside its status', async () => {
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{
fetchImpl: stubFetch(500, {
[EVENTS_HANDLED_HEADER]: '1',
[EVENTS_ERROR_HEADER]: 'handler-threw',
}),
},
);
const transport = new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: stubFetch(500, {
[EVENTS_HANDLED_HEADER]: '1',
[EVENTS_ERROR_HEADER]: 'handler-threw',
}),
});
expect(await transport.send(CALL)).toEqual({
status: 500,
handled: true,
@@ -250,9 +208,7 @@ describe('DispatcherInvokeTransport', () => {
{ fetchImpl: capturingFetch(calls) },
);
await transport.send(CALL);
expect(calls).toEqual([
`http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`,
]);
expect(calls).toEqual([`http://dispatcher/prefix${EVENTS_DISPATCH_PATH}`]);
});
});
@@ -281,12 +237,10 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
/** Every connection taken by a call nobody has answered yet. */
const saturated = async () => {
const { fetchImpl, held } = heldFetch();
// Slots are per call, batched or not; one request per call keeps
// the count readable.
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{ fetchImpl, batchSize: 1 },
{ fetchImpl },
);
const busy = Array.from({ length: INVOKE_CONNECTIONS }, () =>
transport.send(CALL),
@@ -349,534 +303,6 @@ describe('DispatcherInvokeTransport with every connection busy', () => {
});
});
describe('DispatcherInvokeTransport schedule lane', () => {
const heldFetch = () => {
const held: Array<{ answer: () => void; body: string }> = [];
const fetchImpl = ((_url: string, init: RequestInit) =>
new Promise((resolve) => {
held.push({
body: String(init.body),
answer: () =>
resolve(
new Response(null, {
status: 200,
headers: { [EVENTS_HANDLED_HEADER]: '1' },
}),
),
});
})) as unknown as FetchImpl;
return { fetchImpl, held };
};
it('holds schedule runs to their share and keeps serving deliveries', async () => {
const { fetchImpl, held } = heldFetch();
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{ fetchImpl, batchSize: 1 },
);
const scheduled = Array.from(
{ length: SCHEDULE_INVOKE_CONNECTIONS + 5 },
() => transport.send({ ...CALL, lane: 'schedule' as const }),
);
await vi.waitFor(() =>
expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
);
// A delivery waiting behind the schedule queue still gets a connection.
const delivery = transport.send(CALL);
await vi.waitFor(() =>
expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
);
held[SCHEDULE_INVOKE_CONNECTIONS].answer();
await expect(delivery).resolves.toMatchObject({ status: 200 });
// A schedule run's slot goes to the next schedule run waiting.
held[0].answer();
await vi.waitFor(() =>
expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 2),
);
for (let i = 0; i < 4; i++) held[i + 1].answer();
await vi.waitFor(() =>
expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 6),
);
for (const call of held) call.answer();
await Promise.all(scheduled);
});
it('builds the body after the connection wait, so a stamp is the send time', async () => {
const { fetchImpl, held } = heldFetch();
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{ fetchImpl, batchSize: 1 },
);
const busy = Array.from({ length: SCHEDULE_INVOKE_CONNECTIONS }, () =>
transport.send({ ...CALL, lane: 'schedule' as const }),
);
await vi.waitFor(() =>
expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS),
);
let renders = 0;
const stamped = transport.send({
...CALL,
lane: 'schedule',
render: () => `{"n":${++renders}}`,
});
held[0].answer();
await vi.waitFor(() =>
expect(held).toHaveLength(SCHEDULE_INVOKE_CONNECTIONS + 1),
);
expect(held[SCHEDULE_INVOKE_CONNECTIONS].body).toBe('{"n":1}');
for (const call of held) call.answer();
await Promise.all([...busy, stamped]);
});
});
describe('DispatcherInvokeTransport batches', () => {
interface BatchRequest {
url: string;
body: { calls: Array<Record<string, unknown>> };
signal: AbortSignal;
/** Streams one answer line back. */
answer: (line: Record<string, unknown>) => void;
end: () => void;
}
/**
* A fetch whose batch answers stream only when told to; single calls get a
* handled 200 straight away.
*/
const streamingFetch = () => {
const batches: BatchRequest[] = [];
const singles: Array<{ url: string; headers: Record<string, string> }> =
[];
const fetchImpl = (async (url: string, init: RequestInit) => {
if (!url.endsWith(EVENTS_DISPATCH_BATCH_PATH)) {
singles.push({
url,
headers: init.headers as Record<string, string>,
});
return new Response(null, {
status: 200,
headers: { [EVENTS_HANDLED_HEADER]: '1' },
});
}
let controller!: ReadableStreamDefaultController<Uint8Array>;
const stream = new ReadableStream<Uint8Array>({
start: (c) => {
controller = c;
},
});
const encoder = new TextEncoder();
init.signal?.addEventListener('abort', () => {
try {
controller.error(init.signal!.reason);
} catch {
// already closed
}
});
batches.push({
url,
body: JSON.parse(String(init.body)),
signal: init.signal!,
answer: (line) =>
controller.enqueue(
encoder.encode(`${JSON.stringify(line)}\n`),
),
end: () => controller.close(),
});
return new Response(stream, {
status: 200,
headers: { 'content-type': 'application/x-ndjson' },
});
}) as unknown as FetchImpl;
return { fetchImpl, batches, singles };
};
const transportFor = (
fetchImpl: FetchImpl,
options: { batchSize?: number; batchWindowMs?: number } = {},
) =>
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl,
...options,
});
it('sends calls made together as one request, and answers each from its own line', async () => {
const { fetchImpl, batches, singles } = streamingFetch();
const transport = transportFor(fetchImpl);
const first = transport.send({ ...CALL, script: 'evw-a', key: 'k1:a' });
const second = transport.send({
...CALL,
script: 'evw-b',
deployed: true,
body: '{"b":1}',
});
await vi.waitFor(() => expect(batches).toHaveLength(1));
expect(batches[0].url).toBe(
`http://dispatcher${EVENTS_DISPATCH_BATCH_PATH}`,
);
expect(batches[0].body.calls).toEqual([
{
script: 'evw-a',
appUid: 'app-test',
key: 'k1:a',
body: '{}',
waitMs: expect.any(Number),
},
{
script: 'evw-b',
appUid: 'app-test',
key: 'k1:test',
deployed: true,
body: '{"b":1}',
waitMs: expect.any(Number),
},
]);
// What is left of half the timeout, the share a call may wait.
expect(batches[0].body.calls[0].waitMs).toBeLessThanOrEqual(2_500);
expect(batches[0].body.calls[0].waitMs).toBeGreaterThan(2_400);
// Out of order: each line names the call it answers.
batches[0].answer({ i: 1, status: 400, handled: true });
await expect(second).resolves.toEqual({ status: 400, handled: true });
batches[0].answer({
i: 0,
status: 503,
handled: true,
error: 'platform-error',
retryAfter: '3',
});
await expect(first).resolves.toEqual({
status: 503,
handled: true,
error: 'platform-error',
retryAfterMs: 3000,
});
// Every call answered, so the request is let go.
expect(batches[0].signal.aborted).toBe(true);
expect(singles).toEqual([]);
});
it('sends a call with nobody to batch with on the single route', async () => {
const { fetchImpl, batches, singles } = streamingFetch();
const transport = transportFor(fetchImpl);
await expect(transport.send(CALL)).resolves.toMatchObject({
status: 200,
});
expect(batches).toEqual([]);
expect(singles[0].url).toBe(`http://dispatcher${EVENTS_DISPATCH_PATH}`);
});
it('sends a full batch at once, without waiting out the window', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl, {
batchSize: 2,
batchWindowMs: 60_000,
});
const sent = [transport.send(CALL), transport.send(CALL)];
const third = transport.send(CALL);
await vi.waitFor(() => expect(batches).toHaveLength(1));
expect(batches[0].body.calls).toHaveLength(2);
batches[0].answer({ i: 0, status: 200, handled: true });
batches[0].answer({ i: 1, status: 200, handled: true });
await Promise.all(sent);
transport.close();
await expect(third).resolves.toMatchObject({ status: 200 });
});
it('gives each call its own timeout, without failing the rest of the batch', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl);
const quick = transport.send({ ...CALL, timeoutMs: 50 });
const slow = transport.send({ ...CALL, timeoutMs: 5_000 });
await vi.waitFor(() => expect(batches).toHaveLength(1));
batches[0].answer({ i: 0, started: true });
batches[0].answer({ i: 1, started: true });
await expect(quick).resolves.toMatchObject({
status: null,
timedOut: true,
});
expect(batches[0].signal.aborted).toBe(false);
batches[0].answer({ i: 1, status: 200, handled: true });
await expect(slow).resolves.toEqual({ status: 200, handled: true });
});
it('runs a call`s timeout from its started line, not from the send', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl);
let settled = false;
const queued = transport
.send({ ...CALL, timeoutMs: 200 })
.then((answer) => {
settled = true;
return answer;
});
const other = transport.send(CALL);
await vi.waitFor(() => expect(batches).toHaveLength(1));
// Queued behind other calls for longer than its whole timeout.
await new Promise((resolve) => setTimeout(resolve, 250));
expect(settled).toBe(false);
batches[0].answer({ i: 0, started: true });
await new Promise((resolve) => setTimeout(resolve, 100));
expect(settled).toBe(false);
await expect(queued).resolves.toMatchObject({
status: null,
timedOut: true,
});
batches[0].answer({ i: 1, status: 200, handled: true });
await other;
});
it('puts a call that never started on Puter, not on its handler', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl);
const never = transport.send({ ...CALL, timeoutMs: 20 });
const queueTimedOut = transport.send(CALL);
await vi.waitFor(() => expect(batches).toHaveLength(1));
batches[0].answer({ i: 1, status: 503, dispatch: 'queue-timeout' });
await expect(queueTimedOut).resolves.toEqual({
status: null,
error: 'dispatcher: queue-timeout (503)',
dispatchReason: 'queue-timeout',
});
// Its wait plus the grace for the started line to arrive.
await expect(never).resolves.toEqual({
status: null,
error: 'dispatcher: call never started',
});
const client = new EventsWorkerInvokerClient({} as IConfig);
client.setTransport({ send: async () => queueTimedOut });
await expect(client.invoke(REQUEST)).resolves.toMatchObject({
outcome: 'unavailable',
});
});
it('puts a call the stream ended without answering on Puter', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl);
const answered = transport.send(CALL);
const dropped = transport.send(CALL);
await vi.waitFor(() => expect(batches).toHaveLength(1));
batches[0].answer({ i: 0, status: 200, handled: true });
batches[0].end();
await expect(answered).resolves.toEqual({ status: 200, handled: true });
await expect(dropped).resolves.toEqual({
status: null,
error: 'dispatcher: batch ended without an answer',
});
});
it('carries a dispatch failure in a line as that call`s reason', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl);
const missing = transport.send(CALL);
const other = transport.send(CALL);
await vi.waitFor(() => expect(batches).toHaveLength(1));
batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
batches[0].answer({ i: 1, status: 200, handled: true });
await expect(missing).resolves.toEqual({
status: null,
error: 'dispatcher: missing (404)',
dispatchReason: 'missing',
});
await expect(other).resolves.toMatchObject({ status: 200 });
});
it('fails every call of a batch the dispatcher refused, or something else answered', async () => {
for (const [response, expected] of [
[
new Response('forbidden', {
status: 403,
headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'forbidden' },
}),
{
status: null,
error: 'dispatcher: forbidden (403)',
dispatchReason: 'forbidden',
},
],
[
new Response('not found', { status: 404 }),
{
status: null,
error: 'dispatcher: unmarked 404 for a batch',
},
],
] as const) {
const transport = transportFor(
(async () => response) as unknown as FetchImpl,
);
const answers = await Promise.all([
transport.send(CALL),
transport.send(CALL),
]);
expect(answers).toEqual([expected, expected]);
}
});
it('fails every call when the batch request itself fails', async () => {
const transport = transportFor(refusedFetch);
const answers = await Promise.all([
transport.send(CALL),
transport.send(CALL),
]);
expect(answers).toEqual([
{ status: null, error: 'fetch failed' },
{ status: null, error: 'fetch failed' },
]);
});
it('falls back to single calls on a dispatcher without the batch route', async () => {
const urls: string[] = [];
const fetchImpl = (async (url: string) => {
urls.push(url);
if (url.endsWith(EVENTS_DISPATCH_BATCH_PATH))
return new Response('no-such-route', {
status: 404,
headers: {
[EVENTS_DISPATCH_ERROR_HEADER]: 'no-such-route',
},
});
return new Response(null, {
status: 200,
headers: { [EVENTS_HANDLED_HEADER]: '1' },
});
}) as unknown as FetchImpl;
const transport = transportFor(fetchImpl);
const first = await Promise.all([
transport.send(CALL),
transport.send(CALL),
]);
expect(first).toEqual([
{ status: 200, handled: true },
{ status: 200, handled: true },
]);
// The next pair goes straight to the single route.
await Promise.all([transport.send(CALL), transport.send(CALL)]);
expect(urls.map((url) => new URL(url).pathname)).toEqual([
EVENTS_DISPATCH_BATCH_PATH,
EVENTS_DISPATCH_PATH,
EVENTS_DISPATCH_PATH,
EVENTS_DISPATCH_PATH,
EVENTS_DISPATCH_PATH,
]);
});
it('builds each body when the batch goes out', async () => {
const { fetchImpl, batches } = streamingFetch();
const transport = transportFor(fetchImpl);
let renders = 0;
const sent = [
transport.send({ ...CALL, render: () => `{"n":${++renders}}` }),
transport.send(CALL),
];
expect(renders).toBe(0);
await vi.waitFor(() => expect(batches).toHaveLength(1));
expect(batches[0].body.calls[0].body).toBe('{"n":1}');
batches[0].answer({ i: 0, status: 200, handled: true });
batches[0].answer({ i: 1, status: 200, handled: true });
await Promise.all(sent);
});
it('deploys a call`s missing script and retries it, batched with the rest', async () => {
const { fetchImpl, batches } = streamingFetch();
const client = new EventsWorkerInvokerClient({
events: { invokeTimeoutMs: 5_000 },
} as unknown as IConfig);
client.setTransport(transportFor(fetchImpl));
const missHandler = vi.fn(async () => 'deployed' as const);
client.setMissHandler(missHandler);
const missing = client.invoke({ ...REQUEST, script: 'evw-gone' });
const resident = client.invoke(REQUEST);
await vi.waitFor(() => expect(batches).toHaveLength(1));
batches[0].answer({ i: 0, status: 404, dispatch: 'missing' });
batches[0].answer({ i: 1, status: 200, handled: true });
await expect(resident).resolves.toMatchObject({ outcome: 'settled' });
// The retry is alone, so it goes on the single route.
await expect(missing).resolves.toMatchObject({ outcome: 'settled' });
expect(missHandler).toHaveBeenCalledWith('app-test', 'evw-gone', {
lane: undefined,
});
});
it('reads only well-formed answer lines', () => {
expect(parseAnswerLine('not json')).toBeNull();
expect(parseAnswerLine('{"status":200}')).toBeNull();
expect(parseAnswerLine('{"i":-1,"status":200}')).toBeNull();
expect(parseAnswerLine('{"i":0}')).toBeNull();
expect(parseAnswerLine('{"i":1,"started":true}')).toEqual({
i: 1,
started: true,
});
expect(parseAnswerLine('{"i":2,"status":200}')).toEqual({
i: 2,
answer: { status: 200, handled: false },
});
});
});
describe('EventsWorkerInvokerClient invoke options', () => {
it('stamps the event as each attempt goes out', async () => {
const bodies: string[] = [];
const client = makeClient();
client.setTransport({
send: async (call) => {
bodies.push(call.body);
return { status: 200, handled: true };
},
});
let n = 0;
await client.invoke(REQUEST, {
stampEvent: (event) => ({ ...(event as object), sentAt: ++n }),
});
expect(JSON.parse(bodies[0]).event).toEqual({ sentAt: 1 });
});
it('gives a call up as Puter`s when its deploy is put off for another', async () => {
const headersSeen: Array<Record<string, string>> = [];
const client = makeClient();
client.setTransport(
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: missUntilDeployedFetch(headersSeen),
}),
);
client.setMissHandler(async () => 'busy');
await expect(client.invoke(REQUEST)).resolves.toMatchObject({
outcome: 'unavailable',
deployBusy: true,
});
expect(headersSeen).toHaveLength(1);
});
it('tells the miss handler whose call needs the deploy', async () => {
const lanes: unknown[] = [];
const client = makeClient();
client.setTransport(
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: missUntilDeployedFetch([]),
}),
);
client.setMissHandler(async (_appUid, _script, options) => {
lanes.push(options.lane);
return 'deployed';
});
await client.invoke(REQUEST, { lane: 'schedule' });
await client.invoke(REQUEST);
expect(lanes).toEqual(['schedule', undefined]);
});
});
const REQUEST: WorkerInvokeRequest = {
script: 'evw-test',
appUid: 'app-test',
@@ -887,8 +313,7 @@ const REQUEST: WorkerInvokeRequest = {
ctx: {},
};
const makeClient = () =>
new EventsWorkerInvokerClient({} as unknown as IConfig);
const makeClient = () => new EventsWorkerInvokerClient({} as unknown as IConfig);
/** Answers `missing` until it sees the deployed header, then settles. */
const missUntilDeployedFetch = (
@@ -1306,217 +731,3 @@ describe('outcomeForStatus', () => {
},
);
});
describe('EventsWorkerInvokerClient overall deadline', () => {
afterEach(() => {
vi.useRealTimers();
});
const LEASE_MS = EVENTS_INVOKE_TIMEOUT_MS * EVENTS_WORKER_LEASE_FACTOR;
/**
* A transport on a fake clock: each send costs what `cost` says, and
* answers a miss until the deployed header, then a handled 200.
*/
const timedTransport = (cost: (deployed: boolean) => number) => {
const sends: Array<{ deployed: boolean; timeoutMs: number }> = [];
const transport = {
send: async (call: {
deployed?: boolean;
timeoutMs: number;
}): Promise<unknown> => {
const deployed = call.deployed === true;
sends.push({ deployed, timeoutMs: call.timeoutMs });
// A send never outlasts its own timeout.
vi.advanceTimersByTime(
Math.min(cost(deployed), call.timeoutMs * 1.5),
);
return deployed
? { status: 200, handled: true }
: {
status: null,
dispatchReason: 'missing',
error: 'dispatcher: missing (404)',
};
},
};
return { transport, sends };
};
const run = async (
cost: (deployed: boolean) => number,
deployMs: number,
) => {
vi.useFakeTimers({ toFake: ['Date'] });
const { transport, sends } = timedTransport(cost);
const client = makeClient();
client.setTransport(transport as never);
client.setMissHandler(async () => {
vi.advanceTimersByTime(deployMs);
return 'deployed';
});
const started = Date.now();
const result = await client.invoke(REQUEST);
return { result, sends, elapsed: Date.now() - started };
};
it('leaves a fast miss as it was: a full-timeout resend', async () => {
const { result, sends } = await run(() => 100, 2_000);
expect(result).toEqual({ outcome: 'settled', status: 200 });
expect(sends).toEqual([
{ deployed: false, timeoutMs: EVENTS_INVOKE_TIMEOUT_MS },
{ deployed: true, timeoutMs: EVENTS_INVOKE_TIMEOUT_MS },
]);
});
it('shortens the resend after a slow deploy, so the call ends before the lease', async () => {
// A miss that took 12 s, a deploy that took 30 s, and a resend that
// would use everything it is given.
const { result, sends, elapsed } = await run(
(deployed) => (deployed ? 1e9 : 12_000),
30_000,
);
expect(result.outcome).toBe('settled');
expect(sends[1].deployed).toBe(true);
expect(sends[1].timeoutMs).toBeLessThan(EVENTS_INVOKE_TIMEOUT_MS);
expect(elapsed).toBeLessThanOrEqual(
LEASE_MS - EVENTS_DEADLINE_MARGIN_MS,
);
});
it('gives up on a deploy that never settles before the lease, leaving it running', async () => {
vi.useFakeTimers();
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const { transport, sends } = timedTransport(() => 0);
const client = makeClient();
client.setTransport(transport as never);
let rejectDeploy!: (err: Error) => void;
client.setMissHandler(
() =>
new Promise<string>((_resolve, reject) => {
rejectDeploy = reject;
}),
);
let finished: unknown = null;
const pending = client.invoke(REQUEST).then((r) => (finished = r));
// Up to where the call must have ended, and no further.
await vi.advanceTimersByTimeAsync(LEASE_MS - EVENTS_DEADLINE_MARGIN_MS);
expect(finished).toMatchObject({
outcome: 'unavailable',
status: null,
});
await pending;
expect(sends).toHaveLength(1);
// The deploy was not cancelled; its late failure is logged, not thrown.
rejectDeploy(new Error('stalled'));
await vi.advanceTimersByTimeAsync(0);
expect(warn).toHaveBeenCalledWith(
'[events] deploy failed late',
expect.any(Error),
);
warn.mockRestore();
});
it('does not resend when the deploy leaves less than the margin', async () => {
const { result, sends } = await run(
() => 100,
LEASE_MS - EVENTS_DEADLINE_MARGIN_MS * 2 + 1_000,
);
expect(sends).toHaveLength(1);
expect(result).toMatchObject({ outcome: 'unavailable', status: null });
});
});
describe('the holder of a user scope on the wire', () => {
const HOLDER = 'user-uuid-1';
it('sends the holder header on a single invoke, and none for an app scope', async () => {
const seen: Array<Record<string, string>> = [];
const fetchImpl = (async (_url: string, init?: RequestInit) => {
seen.push((init?.headers ?? {}) as Record<string, string>);
return new Response(null, {
status: 200,
headers: { [EVENTS_HANDLED_HEADER]: '1' },
});
}) as unknown as FetchImpl;
const client = makeClient();
client.setTransport(
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl,
batchSize: 1,
}),
);
await client.invoke({ ...REQUEST, holder: HOLDER });
await client.invoke(REQUEST);
expect(seen[0][EVENTS_HOLDER_HEADER]).toBe(HOLDER);
expect(EVENTS_HOLDER_HEADER in seen[1]).toBe(false);
});
it('puts a holder field on a batch call of a user scope only', async () => {
const bodies: Array<{ calls: Array<Record<string, unknown>> }> = [];
const fetchImpl = (async (_url: string, init?: RequestInit) => {
bodies.push(JSON.parse(String(init?.body)));
return new Response(
'{"i":0,"status":200,"handled":true}\n{"i":1,"status":200,"handled":true}\n',
{ status: 200 },
);
}) as unknown as FetchImpl;
const transport = new DispatcherInvokeTransport(
'http://dispatcher',
's',
{ fetchImpl },
);
await Promise.all([
transport.send({ ...CALL, holder: HOLDER }),
transport.send(CALL),
]);
expect(bodies).toHaveLength(1);
expect(bodies[0].calls[0].holder).toBe(HOLDER);
expect('holder' in bodies[0].calls[1]).toBe(false);
});
it('hands the holder to the miss handler, and nothing for an app scope', async () => {
const seen: Array<{ holder?: string }> = [];
const client = makeClient();
client.setTransport(
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl: missUntilDeployedFetch([]),
batchSize: 1,
}),
);
client.setMissHandler(async (_appUid, _script, options) => {
seen.push(options);
return 'deployed';
});
await client.invoke({ ...REQUEST, holder: HOLDER });
await client.invoke(REQUEST);
expect(seen[0].holder).toBe(HOLDER);
expect('holder' in seen[1]).toBe(false);
});
it('keeps the holder out of the body the script sees', async () => {
const bodies: string[] = [];
const client = makeClient();
client.setTransport({
send: async (call) => {
bodies.push(call.body);
expect(call.holder).toBe(HOLDER);
return { status: 200, handled: true };
},
});
await client.invoke({ ...REQUEST, holder: HOLDER });
expect(Object.keys(JSON.parse(bodies[0])).sort()).toEqual([
'ctx',
'event',
'handler',
'token',
]);
});
});
@@ -35,7 +35,6 @@ import { PuterClient } from '../types.js';
* x-puter-internal-auth: <internal secret>
* x-puter-events-script: <script the handler set deploys as>
* x-puter-events-app: <app uid, for the deploy-on-miss callback>
* x-puter-events-holder: <holder uuid of a user scope; absent for app scope>
* x-puter-events-key: <key derived for that script>
* { handler, event, ctx, token }
*
@@ -44,20 +43,6 @@ import { PuterClient } from '../types.js';
*
* On a miss, a handler set here may deploy the script itself and retry once
* with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
*
* Calls that come due within a few milliseconds of each other go out together:
*
* POST <events dispatcher>/invoke-batch
* x-puter-internal-auth: <internal secret>
* { calls: [{ script, appUid, holder?, key, deployed?, body, waitMs }] }
*
* The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
* started: true }` when a call goes to its worker, then `{ i, status, handled?,
* error?, retryAfter?, dispatch? }` when it finishes — the same facts the
* single route carries in its status and headers. A call's timeout runs from
* its `started` line, so time spent queued behind other calls is never the
* handler's. A call still queued `waitMs` after the batch arrived is never
* started and answers `dispatch: "queue-timeout"`.
*/
/** The one route an events worker answers. */
@@ -66,27 +51,6 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
/** The route the events dispatcher takes an invocation on. */
export const EVENTS_DISPATCH_PATH = '/invoke';
/** The route it takes several on, answering each as it finishes. */
export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
/** Calls one batch carries at most. */
export const INVOKE_BATCH_SIZE = 60;
/** How long the first call of a batch waits for others to join it. */
export const INVOKE_BATCH_WINDOW_MS = 5;
/**
* How long a dispatcher without the batch route is sent single calls before the
* batch route is tried again.
*/
const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
/**
* How long past its wait a batched call's `started` line may take to arrive
* before the call is given up as never started.
*/
const BATCH_START_GRACE_MS = 2_000;
/**
* Header the dispatcher marks its own failures with. Anything carrying it never
* reached a handler, so its status says nothing about the delivery.
@@ -128,12 +92,6 @@ const PLATFORM_ERROR_CODES = new Set([
*/
export const EVENTS_DEPLOYED_HEADER = 'x-puter-events-deployed';
/**
* Names the user whose scope a script belongs to, so a miss deploys it from
* their handlers. Absent for an app's own scope.
*/
export const EVENTS_HOLDER_HEADER = 'x-puter-events-holder';
/**
* Dispatch-error reasons this backend can resolve itself by deploying, rather
* than wait on the rehydrate callback finding a backend behind the dispatcher's
@@ -154,90 +112,21 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
*/
const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
/** Whose call asked for the deploy, for the share of deploys it may take. */
export interface WorkerMissOptions {
lane?: InvokeLane;
/**
* Holder uuid of the user scope the script belongs to; absent for app
* scope.
*/
holder?: string;
}
/** What a self-deploy attempt did, and to which app/script it applies. */
export type WorkerMissHandler = (
appUid: string,
script: string,
options: WorkerMissOptions,
) => Promise<'deployed' | string>;
/**
* `work`'s result, or null once `ms` has passed. The work keeps running, and a
* failure after the cutoff is logged since nothing else would see it.
*/
const withinTime = async <T>(
work: Promise<T>,
ms: number,
): Promise<T | null> => {
let timedOut = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const limit = new Promise<null>((resolve) => {
timer = setTimeout(
() => {
timedOut = true;
resolve(null);
},
Math.max(0, ms),
);
});
void (async () => {
try {
await work;
} catch (err) {
if (timedOut) console.warn('[events] deploy failed late', err);
}
})();
try {
return await Promise.race([work, limit]);
} finally {
clearTimeout(timer);
}
};
/** How long a handler has to answer before the attempt is abandoned. */
export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
/**
* How many invoke timeouts a delivery's worker lease lasts. The lease store
* reads this, and `invoke()` keeps a whole call, deploy and resend included,
* inside it.
*/
export const EVENTS_WORKER_LEASE_FACTOR = 2;
/**
* Left unspent at the end of the lease, and the least a resend is worth
* starting with: below it the call is given up as `unavailable` and retried.
*/
export const EVENTS_DEADLINE_MARGIN_MS = 5_000;
/**
* Connections held open to the dispatcher, so a busy fleet is not
* re-handshaking. Also how many calls one transport makes at once: a call past
* that waits for a free one before its invoke timeout starts.
*/
export const INVOKE_CONNECTIONS = 256;
/**
* Of those, how many schedule runs may hold at once, so a burst of schedules
* can never take the connections event deliveries need.
*/
export const SCHEDULE_INVOKE_CONNECTIONS = 64;
/** Whose call this is, for the connection share it may take. */
export type InvokeLane = 'delivery' | 'schedule';
/** What a deploy answers when another deploy of the region holds the way. */
export const DEPLOY_BUSY = 'busy';
const INVOKE_KEEP_ALIVE_MS = 30_000;
/**
@@ -252,8 +141,6 @@ export interface WorkerInvokeRequest {
/** Script the app's current handler set deploys as. */
script: string;
appUid: string;
/** Holder uuid, when the handler lives in a user scope. */
holder?: string;
handler: string;
/** The subscriber's app session token. */
token: string;
@@ -267,8 +154,6 @@ export interface WorkerInvokeRequest {
export interface EventsInvokeCall {
script: string;
appUid: string;
/** Holder uuid of the user scope the script belongs to. */
holder?: string;
/** Value the script compares against its own invoke-key binding. */
key: string;
/** Serialized `{ handler, event, ctx, token }`. */
@@ -276,12 +161,6 @@ export interface EventsInvokeCall {
timeoutMs: number;
/** Set on the retry after this backend deployed the script itself. */
deployed?: boolean;
lane?: InvokeLane;
/**
* Builds `body` again at send time, for a transport that waits for a
* connection first, so a send time stamped into it is the real one.
*/
render?: () => string;
}
/** What one attempt at delivering an invocation came back with. */
@@ -297,8 +176,6 @@ export interface EventsInvokeAnswer {
timedOut?: boolean;
/** From a `retry-after` header in seconds. */
retryAfterMs?: number;
/** A deploy this call needed was put off: another held the way. */
deployBusy?: boolean;
}
/**
@@ -321,8 +198,6 @@ export type WorkerInvokeOutcome =
export interface WorkerInvokeResult {
outcome: WorkerInvokeOutcome;
/** A deploy on miss was put off because another one held the way. */
deployBusy?: boolean;
/** The status the handler answered with, or `null` when it never did. */
status: number | null;
/** Why it could not answer, when nothing did. */
@@ -331,13 +206,6 @@ export interface WorkerInvokeResult {
retryAfterMs?: number;
}
/** How one invocation is sent. */
export interface WorkerInvokeOptions {
lane?: InvokeLane;
/** Applied to the event as each attempt is sent. */
stampEvent?: (event: unknown) => unknown;
}
const NO_TRANSPORT =
'no events dispatcher configured (url and internal secret)';
@@ -368,10 +236,7 @@ export class EventsWorkerInvokerClient extends PuterClient {
this.#dispatcher = null;
}
async invoke(
request: WorkerInvokeRequest,
options: WorkerInvokeOptions = {},
): Promise<WorkerInvokeResult> {
async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
const transport = this.#pickTransport();
if (!transport)
return {
@@ -380,80 +245,32 @@ export class EventsWorkerInvokerClient extends PuterClient {
error: NO_TRANSPORT,
};
const { stampEvent } = options;
const render = (): string =>
JSON.stringify({
const call: EventsInvokeCall = {
script: request.script,
appUid: request.appUid,
key: request.key,
body: JSON.stringify({
handler: request.handler,
event: stampEvent ? stampEvent(request.event) : request.event,
event: request.event,
ctx: request.ctx,
token: request.token,
});
const timeoutMs = this.#timeoutMs();
// The whole call, deploy and resend included, ends before the lease a
// delivery holds would let it be claimed again mid-run.
// A short configured timeout gets a proportionally short margin.
const margin = Math.min(EVENTS_DEADLINE_MARGIN_MS, timeoutMs / 2);
const deadline =
Date.now() + timeoutMs * EVENTS_WORKER_LEASE_FACTOR - margin;
const send = (
deployed: boolean,
sendTimeoutMs = timeoutMs,
): Promise<EventsInvokeAnswer> =>
transport.send({
script: request.script,
appUid: request.appUid,
...(request.holder ? { holder: request.holder } : {}),
key: request.key,
body: render(),
timeoutMs: sendTimeoutMs,
...(deployed ? { deployed: true } : {}),
...(options.lane ? { lane: options.lane } : {}),
...(stampEvent ? { render } : {}),
});
// A send may wait for a connection for half its timeout before the
// run's own timeout starts, so it is given what is left divided by that.
const resend = (): Promise<EventsInvokeAnswer> | null => {
const remaining = deadline - Date.now();
if (remaining < margin) return null;
return send(
true,
Math.floor(
Math.min(
timeoutMs,
remaining / (1 + CONNECTION_WAIT_SHARE),
),
),
);
}),
timeoutMs: this.#timeoutMs(),
};
const outOfTime = (): WorkerInvokeResult => ({
outcome: 'unavailable',
status: null,
error: 'out of time to resend after deploy',
});
let result = await send(false);
let result = await transport.send(call);
if (this.#missHandler && selfDeployable(result)) {
// The deploy may be shared with other callers, so it is left
// running when this call runs out of time for it.
const outcome = await withinTime(
this.#missHandler(request.appUid, request.script, {
lane: options.lane,
...(request.holder ? { holder: request.holder } : {}),
}),
deadline - Date.now() - margin,
const outcome = await this.#missHandler(
request.appUid,
request.script,
);
if (outcome === null) return outOfTime();
if (outcome !== 'deployed')
return {
outcome: 'unavailable',
status: null,
error: `deploy: ${outcome}`,
...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
};
const resent = await resend();
if (!resent) return outOfTime();
result = resent;
result = await transport.send({ ...call, deployed: true });
} else if (
result.status === null &&
result.dispatchReason &&
@@ -461,19 +278,10 @@ export class EventsWorkerInvokerClient extends PuterClient {
) {
// No deploy to trigger — just give propagation the one retry
// it usually needs, without spending an upload on it.
const resent = await resend();
if (!resent) return outOfTime();
result = resent;
result = await transport.send({ ...call, deployed: true });
}
const { status, handled, error, timedOut, retryAfterMs } = result;
if (result.deployBusy)
return {
outcome: 'unavailable',
status: null,
error,
deployBusy: true,
};
// Past the timeout the handler was most likely still running, which
// is its own doing; any other missing answer never reached it.
if (status === null)
@@ -505,19 +313,9 @@ export class EventsWorkerInvokerClient extends PuterClient {
// a misconfiguration worth reporting once per attempt rather than a
// round trip.
if (!dispatcherUrl || !internalSecret) return null;
const { invokeBatchSize, invokeBatchWindowMs } =
this.config.events ?? {};
this.#dispatcher ??= new DispatcherInvokeTransport(
dispatcherUrl,
internalSecret,
{
...(invokeBatchSize !== undefined
? { batchSize: invokeBatchSize }
: {}),
...(invokeBatchWindowMs !== undefined
? { batchWindowMs: invokeBatchWindowMs }
: {}),
},
);
return this.#dispatcher;
}
@@ -530,63 +328,28 @@ export class EventsWorkerInvokerClient extends PuterClient {
}
}
/** A call waiting in, or sent with, the current batch. */
interface BatchedCall {
call: EventsInvokeCall;
resolve: (answer: EventsInvokeAnswer) => void;
/** When its wait for a connection began, for what is left of it. */
waitingSince: number;
}
export interface DispatcherInvokeTransportOptions {
fetchImpl?: typeof undiciFetch;
/** Calls per batch; 1 sends every call on its own. */
batchSize?: number;
batchWindowMs?: number;
}
/** Reaches an events worker the only way production can: through the dispatcher. */
export class DispatcherInvokeTransport implements EventsInvokeTransport {
readonly #url: string;
readonly #secret: string;
readonly #fetchImpl: typeof undiciFetch;
readonly #batchSize: number;
readonly #batchWindowMs: number;
#agent: Agent | null = null;
/** Calls holding a connection slot. */
#active = 0;
/** Of those, schedule runs. */
#scheduleActive = 0;
/** Calls waiting for one, longest-waiting first, with their lane. */
readonly #waiting = new Map<() => void, InvokeLane>();
/** Calls gathered for the next batch. */
#batch: BatchedCall[] = [];
#batchTimer: ReturnType<typeof setTimeout> | null = null;
/** Until when the dispatcher is taken not to have the batch route. */
#batchUnsupportedUntil = 0;
/** Calls waiting for one, longest-waiting first. */
readonly #waiting = new Set<() => void>();
constructor(
dispatcherUrl: string,
secret: string,
options: DispatcherInvokeTransportOptions = {},
options: { fetchImpl?: typeof undiciFetch } = {},
) {
// Joined as a string: `new URL(path, base)` resolves an absolute path
// against the base's origin, dropping any path prefix it carries.
this.#url = dispatcherUrl.replace(/\/$/, '');
this.#url = dispatcherUrl;
this.#secret = secret;
this.#fetchImpl = options.fetchImpl ?? undiciFetch;
this.#batchSize = Math.max(
1,
Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
);
this.#batchWindowMs = Math.max(
0,
options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
);
}
close(): void {
this.#flush();
void this.#agent?.close().catch(() => {});
this.#agent = null;
}
@@ -594,71 +357,29 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
// No `timedOut`: the handler never got this call, so the wait is not
// its failure.
const lane = call.lane ?? 'delivery';
const waitingSince = Date.now();
if (
!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
)
if (!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE)))
return {
status: null,
error: 'no free connection to the events dispatcher',
};
try {
if (this.#batchSize === 1) return await this.#sendOne(call);
return await new Promise<EventsInvokeAnswer>((resolve) => {
this.#batch.push({ call, resolve, waitingSince });
if (this.#batch.length >= this.#batchSize) this.#flush();
else
this.#batchTimer ??= setTimeout(
() => this.#flush(),
this.#batchWindowMs,
);
});
} finally {
this.#release(lane);
}
}
/** Send what has gathered: alone when it is one call, else as a batch. */
#flush(): void {
if (this.#batchTimer) clearTimeout(this.#batchTimer);
this.#batchTimer = null;
const batch = this.#batch;
this.#batch = [];
if (batch.length === 0) return;
if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
for (const { call, resolve } of batch)
void this.#sendOne(call).then(resolve);
return;
}
void this.#sendBatch(batch);
}
/** One call on the single route. */
async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
try {
const response = await this.#fetchImpl(
`${this.#url}${EVENTS_DISPATCH_PATH}`,
{
method: 'POST',
headers: {
'content-type': 'application/json',
'x-puter-internal-auth': this.#secret,
'x-puter-events-script': call.script,
'x-puter-events-app': call.appUid,
...(call.holder
? { [EVENTS_HOLDER_HEADER]: call.holder }
: {}),
'x-puter-events-key': call.key,
...(call.deployed
? { [EVENTS_DEPLOYED_HEADER]: '1' }
: {}),
},
body: call.render ? call.render() : call.body,
signal: AbortSignal.timeout(call.timeoutMs),
dispatcher: this.#dispatcherFor(),
// `new URL(path, base)` resolves an absolute path against the
// base's origin, dropping any path prefix `base` already carries.
const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
const response = await this.#fetchImpl(url, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-puter-internal-auth': this.#secret,
'x-puter-events-script': call.script,
'x-puter-events-app': call.appUid,
'x-puter-events-key': call.key,
...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
},
);
body: call.body,
signal: AbortSignal.timeout(call.timeoutMs),
dispatcher: this.#dispatcherFor(),
});
// The body is not read: a handler answers with a status, and
// whatever else it writes is its own business.
void response.body?.cancel().catch(() => {});
@@ -669,182 +390,24 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
EVENTS_DISPATCH_ERROR_HEADER,
);
if (dispatchError !== null)
return dispatchAnswer(dispatchError, response.status);
return {
status: null,
error: `dispatcher: ${dispatchError} (${response.status})`,
dispatchReason: dispatchError,
};
return answerFrom(response);
} catch (err) {
return failedAnswer(err);
} finally {
this.#release();
}
}
/**
* Several calls in one request. A call may wait in the dispatcher's queue
* for whatever is left of its connection wait, and its timeout runs from
* its `started` line; the request itself is dropped once every call has an
* answer.
*/
async #sendBatch(batch: BatchedCall[]): Promise<void> {
const hangUp = new AbortController();
let open = batch.length;
const answered = batch.map(() => false);
const timers: ReturnType<typeof setTimeout>[] = [];
const settle = (i: number, answer: EventsInvokeAnswer): void => {
if (answered[i]) return;
answered[i] = true;
clearTimeout(timers[i]);
batch[i].resolve(answer);
if (--open === 0) hangUp.abort();
};
const settleRest = (answer: EventsInvokeAnswer): void => {
for (let i = 0; i < batch.length; i++) settle(i, answer);
};
const now = Date.now();
const waits = batch.map(({ call, waitingSince }) =>
Math.max(
0,
Math.floor(
call.timeoutMs * CONNECTION_WAIT_SHARE -
(now - waitingSince),
),
),
);
const body = JSON.stringify({
calls: batch.map(({ call }, i) => ({
script: call.script,
appUid: call.appUid,
...(call.holder ? { holder: call.holder } : {}),
key: call.key,
...(call.deployed ? { deployed: true } : {}),
body: call.render ? call.render() : call.body,
waitMs: waits[i],
})),
});
const arm = (
i: number,
ms: number,
answer: () => EventsInvokeAnswer,
) => {
clearTimeout(timers[i]);
timers[i] = setTimeout(() => settle(i, answer()), ms);
timers[i].unref?.();
};
// No `timedOut`: a call that never started never reached a handler.
batch.forEach((_, i) =>
arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
status: null,
error: 'dispatcher: call never started',
})),
);
const started = (i: number): void => {
if (!answered[i])
arm(i, batch[i].call.timeoutMs, () =>
failedAnswer(timeoutError()),
);
};
const longest = Math.max(
...batch.map(
({ call }, i) =>
waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
),
);
try {
const response = await this.#fetchImpl(
`${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
{
method: 'POST',
headers: {
'content-type': 'application/json',
'x-puter-internal-auth': this.#secret,
},
body,
signal: AbortSignal.any([
hangUp.signal,
AbortSignal.timeout(longest),
]),
dispatcher: this.#dispatcherFor(),
},
);
const dispatchError = response.headers.get(
EVENTS_DISPATCH_ERROR_HEADER,
);
if (dispatchError === 'no-such-route') {
// A dispatcher from before the batch route: nothing ran, so
// every call goes again on its own.
void response.body?.cancel().catch(() => {});
this.#batchUnsupportedUntil =
Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
batch.forEach(({ call }, i) => {
if (!answered[i])
void this.#sendOne(call).then((a) => settle(i, a));
});
return;
}
if (dispatchError !== null || !response.ok) {
void response.body?.cancel().catch(() => {});
// An unmarked failure (an edge 404, a WAF 403) reads as
// Puter's, never as any handler's answer.
settleRest(
dispatchError !== null
? dispatchAnswer(dispatchError, response.status)
: {
status: null,
error: `dispatcher: unmarked ${response.status} for a batch`,
},
);
return;
}
let buffered = '';
const read = (line: string): void => {
if (line.trim() === '') return;
const parsed = parseAnswerLine(line);
if (!parsed || parsed.i >= batch.length) return;
if ('started' in parsed) started(parsed.i);
else settle(parsed.i, parsed.answer);
};
const decoder = new TextDecoder();
for await (const chunk of response.body ?? []) {
buffered += decoder.decode(chunk, { stream: true });
let newline = buffered.indexOf('\n');
while (newline !== -1) {
read(buffered.slice(0, newline));
buffered = buffered.slice(newline + 1);
newline = buffered.indexOf('\n');
}
}
read(buffered);
settleRest({
status: null,
error: 'dispatcher: batch ended without an answer',
});
} catch (err) {
// Every call has its answer and the request was dropped on purpose.
if (open === 0) return;
settleRest(failedAnswer(err));
}
}
/** Whether a call on `lane` may take a connection now. */
#free(lane: InvokeLane): boolean {
return (
this.#active < INVOKE_CONNECTIONS &&
(lane !== 'schedule' ||
this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
);
}
#take(lane: InvokeLane): void {
this.#active++;
if (lane === 'schedule') this.#scheduleActive++;
}
/** A connection slot, or false once `waitMs` passes without one. */
#acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
if (this.#free(lane)) {
this.#take(lane);
#acquire(waitMs: number): Promise<boolean> {
if (this.#active < INVOKE_CONNECTIONS) {
this.#active++;
return Promise.resolve(true);
}
return new Promise((resolve) => {
@@ -857,24 +420,19 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
resolve(false);
}, waitMs);
timer.unref?.();
this.#waiting.set(granted, lane);
this.#waiting.add(granted);
});
}
/**
* Give the slot back, then hand one to the longest waiter it can go to: a
* schedule waiter at a full schedule share doesn't hold up a delivery.
*/
#release(lane: InvokeLane): void {
this.#active--;
if (lane === 'schedule') this.#scheduleActive--;
for (const [granted, waiting] of this.#waiting) {
if (!this.#free(waiting)) continue;
this.#waiting.delete(granted);
this.#take(waiting);
granted();
/** Hand the slot to the longest waiter, or give it back. */
#release(): void {
const next = this.#waiting.values().next().value;
if (next) {
this.#waiting.delete(next);
next();
return;
}
this.#active--;
}
/** One pool per process, built on first use and closed with the server. */
@@ -888,65 +446,6 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
}
}
/** What a failure the dispatcher marked as its own reads as. */
const dispatchAnswer = (
reason: string,
status: number,
): EventsInvokeAnswer => ({
status: null,
error: `dispatcher: ${reason} (${status})`,
dispatchReason: reason,
});
/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
const timeoutError = (): Error =>
new DOMException(
'The operation was aborted due to timeout',
'TimeoutError',
);
/** One line of a batch answer, or null when it is not one. */
export const parseAnswerLine = (
line: string,
):
| { i: number; started: true }
| { i: number; answer: EventsInvokeAnswer }
| null => {
let raw: unknown;
try {
raw = JSON.parse(line);
} catch {
return null;
}
if (!raw || typeof raw !== 'object') return null;
const { i, started, status, handled, error, retryAfter, dispatch } =
raw as Record<string, unknown>;
if (!Number.isInteger(i) || (i as number) < 0) return null;
if (started === true) return { i: i as number, started: true };
if (typeof status !== 'number') return null;
if (typeof dispatch === 'string')
return { i: i as number, answer: dispatchAnswer(dispatch, status) };
return {
i: i as number,
answer: answerFrom({
status,
headers: {
get: (name) => {
if (name === EVENTS_HANDLED_HEADER)
return handled === true ? '1' : null;
if (name === EVENTS_ERROR_HEADER)
return typeof error === 'string' ? error : null;
if (name === 'retry-after')
return typeof retryAfter === 'string'
? retryAfter
: null;
return null;
},
},
}),
};
};
/** Whether deploying the script is what fixes this answer. */
const selfDeployable = (result: EventsInvokeAnswer): boolean =>
result.status === null
@@ -1416,6 +1416,7 @@ describe('AuthController.handleLogin', () => {
is_user_token: true,
});
expect(body.user.directories).toBeTypeOf('object');
expect(Array.isArray(body.user.taskbar_items)).toBe(true);
});
// A deleted row still answering from cache reaches the session INSERT.
@@ -19,7 +19,6 @@
import type { Request, Response } from 'express';
import { Controller, Post } from '../../core/http/decorators.js';
import { schedulesUnavailable } from '../../core/schedules.js';
import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
import type { ForwardBatch } from '../../services/events/forwardQueue.js';
import { PuterController } from '../types.js';
@@ -109,42 +108,6 @@ export class BroadcastController extends PuterController {
);
res.status(200).json({ ok: true, ...reply });
}
/**
* Schedule changes a region made for schedules this one owns, handed to
* `services.schedules`; `events_disabled` on a server without it.
*/
@Post('/events/schedules', { subdomain: '*' })
async schedules(req: Request, res: Response): Promise<void> {
const broadcast = this.services.broadcast as unknown as
BroadcastService | undefined;
if (!broadcast) {
res.status(503).json({
error: { message: 'Broadcast service not registered' },
});
return;
}
const verified = await broadcast.verifySignedRequest(
req.rawBody,
broadcastHeaders(req),
);
if (!verified.ok) {
res.status(verified.status ?? 400).json({
error: { message: verified.message ?? 'Bad request' },
});
return;
}
if (verified.info?.ignored) {
res.status(200).json({ ok: true, ...verified.info });
return;
}
const schedules = this.services.schedules;
if (!schedules) throw schedulesUnavailable();
const reply = await schedules.receivePeerChanges(req.body);
res.status(200).json({ ok: true, ...reply });
}
}
const broadcastHeaders = (req: Request) => {
@@ -25,13 +25,18 @@ const ALLOWED_SORT_BY = ['name', 'size', 'modified', 'type'];
const ALLOWED_SORT_ORDER = ['asc', 'desc'];
/**
* Folder view preference routes: an explorer window's layout and sort order,
* kept on the folder's fsentry.
* Desktop/UI preference routes.
*
* Two categories:
*
* - User-level: desktop background, taskbar items (UserStore)
* - Folder-level: layout, sort_by/sort_order (fsentries table)
*/
/**
* Folder preference writes — layout, sort order. The GUI fires them on direct
* user action, so a per-minute ceiling well above human speed is enough to
* catch a stuck client.
* Desktop preference writes — background, taskbar, layout, sort order. All four
* persist to the user row on every call, and the GUI fires them on direct user
* action, so a per-minute ceiling well above human speed is enough to catch a
* stuck client.
*/
const PREFERENCE_WRITE_LIMIT = {
scope: 'desktop-preference',
@@ -42,6 +47,89 @@ const PREFERENCE_WRITE_LIMIT = {
export class DesktopController extends PuterController {
registerRoutes(router) {
// -- Desktop background --------------------------------------
router.post(
'/set-desktop-bg',
{
subdomain: 'api',
requireUserActor: true,
allowFullAccessToken: true,
rateLimit: PREFERENCE_WRITE_LIMIT,
},
async (req, res) => {
const { url, color, fit } = req.body ?? {};
const patch = {};
if (url !== undefined) {
if (url !== null && typeof url !== 'string') {
throw new HttpError(
400,
'`url` must be a string or null',
{ legacyCode: 'bad_request' },
);
}
patch.desktop_bg_url = url;
}
if (color !== undefined) {
if (color !== null && typeof color !== 'string') {
throw new HttpError(
400,
'`color` must be a string or null',
{ legacyCode: 'bad_request' },
);
}
patch.desktop_bg_color = color;
}
if (fit !== undefined) {
if (fit !== null && typeof fit !== 'string') {
throw new HttpError(
400,
'`fit` must be a string or null',
{ legacyCode: 'bad_request' },
);
}
patch.desktop_bg_fit = fit;
}
if (Object.keys(patch).length === 0) {
throw new HttpError(400, 'No fields provided', {
legacyCode: 'bad_request',
});
}
await this.stores.user.update(req.actor.user.id, patch);
res.json({});
},
);
// -- Taskbar items -------------------------------------------
router.post(
'/update-taskbar-items',
{
subdomain: 'api',
requireUserActor: true,
allowFullAccessToken: true,
rateLimit: PREFERENCE_WRITE_LIMIT,
},
async (req, res) => {
const { items } = req.body ?? {};
if (!Array.isArray(items)) {
throw new HttpError(
400,
'Missing or invalid `items` array',
{ legacyCode: 'bad_request' },
);
}
await this.stores.user.update(req.actor.user.id, {
taskbar_items: JSON.stringify(items),
});
res.json({});
},
);
// -- Folder layout -------------------------------------------
router.post(
@@ -130,6 +130,150 @@ const callRoute = async (
});
};
// ── /set-desktop-bg ─────────────────────────────────────────────────
describe('DesktopController POST /set-desktop-bg', () => {
it('persists url/color/fit on the user row', async () => {
const { actor, userId } = await makeUser();
const { res, captured } = makeRes();
await callRoute(
'post',
'/set-desktop-bg',
makeReq({
body: {
url: 'https://cdn.test/wallpaper.png',
color: '#000000',
fit: 'cover',
},
actor,
}),
res,
);
expect(captured.body).toEqual({});
const refreshed = await server.stores.user.getById(userId);
expect(refreshed?.desktop_bg_url).toBe(
'https://cdn.test/wallpaper.png',
);
expect(refreshed?.desktop_bg_color).toBe('#000000');
expect(refreshed?.desktop_bg_fit).toBe('cover');
});
it('persists only the supplied fields (partial update)', async () => {
const { actor, userId } = await makeUser();
const { res } = makeRes();
await callRoute(
'post',
'/set-desktop-bg',
makeReq({ body: { color: '#ffffff' }, actor }),
res,
);
const refreshed = await server.stores.user.getById(userId);
expect(refreshed?.desktop_bg_color).toBe('#ffffff');
// Untouched fields remain at their column default.
expect(refreshed?.desktop_bg_url).toBeFalsy();
expect(refreshed?.desktop_bg_fit).toBeFalsy();
});
it('passes through `null` to clear a field', async () => {
const { actor, userId } = await makeUser();
// First populate, then clear.
const populate = makeRes();
await callRoute(
'post',
'/set-desktop-bg',
makeReq({
body: { url: 'https://cdn.test/x.png' },
actor,
}),
populate.res,
);
const before = await server.stores.user.getById(userId);
expect(before?.desktop_bg_url).toBe('https://cdn.test/x.png');
const clear = makeRes();
await callRoute(
'post',
'/set-desktop-bg',
makeReq({ body: { url: null }, actor }),
clear.res,
);
const after = await server.stores.user.getById(userId);
expect(after?.desktop_bg_url).toBeNull();
});
it('throws 400 when url is not a string or null', async () => {
const { actor } = await makeUser();
const { res } = makeRes();
await expect(
callRoute(
'post',
'/set-desktop-bg',
makeReq({ body: { url: 123 }, actor }),
res,
),
).rejects.toMatchObject({ statusCode: 400 });
});
it('throws 400 when no fields are provided', async () => {
const { actor } = await makeUser();
const { res } = makeRes();
await expect(
callRoute(
'post',
'/set-desktop-bg',
makeReq({ body: {}, actor }),
res,
),
).rejects.toMatchObject({ statusCode: 400 });
});
});
// ── /update-taskbar-items ───────────────────────────────────────────
describe('DesktopController POST /update-taskbar-items', () => {
it('persists items as a JSON-encoded string', async () => {
const { actor, userId } = await makeUser();
const items = [{ name: 'editor' }, { name: 'browser' }];
const { res } = makeRes();
await callRoute(
'post',
'/update-taskbar-items',
makeReq({ body: { items }, actor }),
res,
);
const refreshed = await server.stores.user.getById(userId);
expect(refreshed?.taskbar_items).toBe(JSON.stringify(items));
});
it('throws 400 when items is missing', async () => {
const { actor } = await makeUser();
const { res } = makeRes();
await expect(
callRoute(
'post',
'/update-taskbar-items',
makeReq({ body: {}, actor }),
res,
),
).rejects.toMatchObject({ statusCode: 400 });
});
it('throws 400 when items is not an array', async () => {
const { actor } = await makeUser();
const { res } = makeRes();
await expect(
callRoute(
'post',
'/update-taskbar-items',
makeReq({ body: { items: 'oops' }, actor }),
res,
),
).rejects.toMatchObject({ statusCode: 400 });
});
});
// ── /set_layout ─────────────────────────────────────────────────────
describe('DesktopController POST /set_layout', () => {
@@ -25,11 +25,6 @@ import { bodyRecord } from '../../core/http/requestBody.js';
import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
import {
SCHEDULE_LIST_LIMIT_CAP,
schedulesUnavailable,
type ScheduleServiceContract,
} from '../../core/schedules.js';
import { normalizeLimit } from '../../util/pagination.js';
import { PuterController } from '../types.js';
import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -64,8 +59,6 @@ const EVENTS_SCRIPT_REGEX = new RegExp(`^${EVENTS_WORKER_PREFIX}[a-f0-9]{32}$`);
/** Matches the edge dispatcher's own validation of the app uid it forwards. */
const APP_UID_REGEX = /^app-[A-Za-z0-9_-]{1,64}$/;
/** A user uuid. */
const HOLDER_REGEX = /^[A-Za-z0-9-]{1,64}$/;
@Controller('/events')
export class EventsController extends PuterController {
@@ -134,8 +127,7 @@ export class EventsController extends PuterController {
*
* Stateless: no subscription, no stored position, nothing written. The
* cursor comes back to the caller and goes out again as `cursor` (or
* `after`, the name it first shipped under). `from=end` with no cursor
* starts at the newest row instead of the oldest.
* `after`, the name it first shipped under).
*/
@Get('/fetch', {
subdomain: 'api',
@@ -146,16 +138,9 @@ export class EventsController extends PuterController {
async fetch(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
const query = (req.query ?? {}) as Record<string, unknown>;
const from = query.from as 'start' | 'end' | undefined;
if (from !== undefined && from !== 'start' && from !== 'end') {
throw new HttpError(400, '`from` must be `start` or `end`', {
legacyCode: 'bad_request',
});
}
const page = await this.services.events.fetchMissed(actor, {
subject: typeof query.subject === 'string' ? query.subject : '',
from,
cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
after: typeof query.after === 'string' ? query.after : undefined,
limit: normalizeLimit(query.limit, {
@@ -184,88 +169,6 @@ export class EventsController extends PuterController {
res.json({});
}
// -- Schedules ---------------------------------------------------
//
// An app's handler run on a timer, served by `services.schedules` when a
// server has it and answered `events_disabled` when it doesn't. Creating
// one needs an app context; listing, reading and removing are scoped the
// way the durable verbs are. The create and remove verbs spend the
// subscribe and unsubscribe budgets in the service, shared with those
// verbs.
#schedules(): ScheduleServiceContract {
const schedules = this.services.schedules;
if (!schedules) throw schedulesUnavailable();
return schedules;
}
/** POST /events/schedules — create a schedule for the calling app. */
@Post('/schedules', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
})
async createSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
}
/** GET /events/schedules — the caller's schedules, one page at a time. */
@Get('/schedules', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
rateLimit: EVENTS_LIST_LIMIT,
})
async listSchedules(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
const query = (req.query ?? {}) as Record<string, unknown>;
const page = await this.#schedules().list(actor, {
limit: normalizeLimit(query.limit, {
cap: SCHEDULE_LIST_LIMIT_CAP,
}),
cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
includeTotal: query.includeTotal === 'true',
});
res.json({
items: page.items,
...(page.cursor ? { cursor: page.cursor } : {}),
...(page.total !== undefined ? { total: page.total } : {}),
});
}
/**
* POST /events/schedules/remove — end one schedule. An id the caller does
* not hold reads as absent.
*/
@Post('/schedules/remove', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
})
async removeSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
res.json({});
}
/**
* GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
* caller does not hold reads as absent.
*/
@Get('/schedules/:scheduleId', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
rateLimit: EVENTS_LIST_LIMIT,
})
async getSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
res.json(await this.#schedules().get(actor, req.params.scheduleId));
}
// -- Cross-user key-value handles --------------------------------
/**
@@ -412,7 +315,6 @@ export class EventsController extends PuterController {
handlers: await this.services.events.listHandlers(actor, {
appUid:
typeof query.appUid === 'string' ? query.appUid : undefined,
scope: query.scope,
}),
});
}
@@ -511,15 +413,6 @@ export class EventsController extends PuterController {
throw new HttpError(400, 'Missing or invalid `script`/`appUid`', {
legacyCode: 'bad_request',
});
// Absent or empty is the app's own scope.
const holder = body.holder === undefined ? '' : body.holder;
if (
typeof holder !== 'string' ||
!(holder === '' || HOLDER_REGEX.test(holder))
)
throw new HttpError(400, 'Invalid `holder`', {
legacyCode: 'bad_request',
});
const deployer = this.#eventsWorkerDeployer();
if (!deployer.enabled) {
@@ -527,25 +420,15 @@ export class EventsController extends PuterController {
return;
}
const scope = await deployer.scopeFor(appUid, holder);
if (!scope) {
res.status(404).json({ deployed: false, reason: 'no-holder' });
return;
}
const outcome = await deployer.ensure(scope, script);
const outcome = await deployer.ensure(appUid, script);
if (outcome === 'deployed') {
res.json({ deployed: true });
return;
}
// `failed`, `throttled` and `busy` are worth another attempt: the rest
// say this script is not what the app's handlers currently are.
// `failed` and `throttled` are worth another attempt: the rest say
// this script is not what the app's handlers currently are.
res.status(
outcome === 'failed' ||
outcome === 'throttled' ||
outcome === 'busy'
? 502
: 404,
outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
).json({
deployed: false,
reason: outcome,
@@ -566,22 +449,14 @@ export class EventsController extends PuterController {
override onServerStart(): void {
const deployer = this.#eventsWorkerDeployer();
if (!deployer.enabled) return;
this.services.events.useEagerDeploy((scope, script) =>
deployer.deployEagerly(scope, script),
);
if (this.config.workers?.localServer) {
this.services.events.useWorkerTransport(
new LocalEventsInvokeTransport(this.services, deployer),
);
return;
}
this.services.events.useWorkerMissHandler(
async (appUid, script, options) => {
const scope = await deployer.scopeFor(appUid, options.holder);
return scope
? deployer.ensure(scope, script, options)
: 'no-holder';
},
this.services.events.useWorkerMissHandler((appUid, script) =>
deployer.ensure(appUid, script),
);
}
+7 -69
View File
@@ -34,46 +34,23 @@ import {
EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP,
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
EVENTS_HANDLER_PUBLISH_LIMIT,
EVENTS_HANDLER_SOURCE_MAX_BYTES,
EVENTS_HANDLERS_PER_APP,
EVENTS_KV_HANDLES_PER_APP,
EVENTS_KV_HANDLES_PER_USER,
EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT,
EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT,
EVENTS_SINGLE_DELIVERY_LIMIT,
EVENTS_USER_HANDLERS_PER_SCOPE,
EVENTS_USER_SCOPE_SOURCE_MAX_BYTES,
EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_INVOCATION_LIMIT,
EVENTS_WORKER_SOURCE_MAX_BYTES,
RENTED_UNITS_PER_USER,
limitFor,
tiered,
type TieredLimit,
} from './limits.js';
const tiers: Array<[string, TieredLimit]> = [
[
'durable subscriptions per account',
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
],
['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
];
describe('tiered', () => {
it('gives paid plans the base, and the free and temporary tiers their own', () => {
const limit = tiered(30, 20, 10);
expect(limitFor(limit, null)).toBe(30);
expect(limitFor(limit, 'some_plan')).toBe(30);
expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
});
});
describe('the tiered subscription quotas', () => {
it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
for (const n of Object.values(tier.bySubscription))
@@ -128,9 +105,9 @@ describe('the tiered subscription quotas', () => {
});
it('holds an unrecognised plan to the paid base', () => {
expect(
limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
);
expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
);
@@ -154,9 +131,9 @@ describe('the delivery budgets', () => {
expect(
limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
).toBe(matched);
expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
evaluated,
);
expect(
limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
).toBe(evaluated);
expect(evaluated).toBeGreaterThanOrEqual(matched);
}
});
@@ -192,42 +169,3 @@ describe('the delivery budgets', () => {
}
});
});
describe('the handler limits', () => {
it('pins the app-scope numbers the quotas page lists', () => {
expect(EVENTS_HANDLERS_PER_APP).toBe(100);
expect(EVENTS_HANDLER_SOURCE_MAX_BYTES).toBe(64 * 1024);
expect(EVENTS_WORKER_SOURCE_MAX_BYTES).toBe(5 * 1024 * 1024);
expect(EVENTS_WORKER_DEPLOYS_PER_HOUR).toBe(30);
expect(EVENTS_HANDLER_PUBLISH_LIMIT.limit).toBe(60);
expect(EVENTS_HANDLER_PUBLISH_LIMIT.window).toBe(60_000);
});
it('pins the user-scope numbers the quotas page lists', () => {
expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBe(20);
expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBe(1024 * 1024);
expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBe(10);
});
it('keeps a user scope tighter than an app scope', () => {
expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBeLessThan(
EVENTS_HANDLERS_PER_APP,
);
expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBeLessThan(
EVENTS_WORKER_SOURCE_MAX_BYTES,
);
expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBeLessThan(
EVENTS_WORKER_DEPLOYS_PER_HOUR,
);
});
it('pins the rented-unit cap by plan', () => {
expect(RENTED_UNITS_PER_USER.limit).toBe(100);
expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_FREE_SUBSCRIPTION)).toBe(
16,
);
expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_TEMP_SUBSCRIPTION)).toBe(
16,
);
});
});
+13 -32
View File
@@ -53,12 +53,7 @@ export interface TieredLimit {
bySubscription: Record<string, number>;
}
/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
export const tiered = (
paid: number,
free: number,
temp: number,
): TieredLimit => ({
const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
limit: paid,
bySubscription: {
[DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -269,12 +264,6 @@ export const KV_SHARE_NOTIFY_LIMITS = [
*/
export const EVENTS_HANDLERS_PER_APP = 100;
/**
* Handlers one user may hold in their own scope of one app. Each is a script of
* its own, so the cap is tighter than the app's.
*/
export const EVENTS_USER_HANDLERS_PER_SCOPE = 20;
/** Longest a serialized handler may be. */
export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
@@ -286,9 +275,6 @@ export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
*/
export const EVENTS_WORKER_SOURCE_MAX_BYTES = 5 * 1024 * 1024;
/** Source one user scope may hold, all its handlers combined. */
export const EVENTS_USER_SCOPE_SOURCE_MAX_BYTES = 1024 * 1024;
/**
* Deploys one app's events worker may trigger per hour.
*
@@ -298,16 +284,6 @@ export const EVENTS_USER_SCOPE_SOURCE_MAX_BYTES = 1024 * 1024;
*/
export const EVENTS_WORKER_DEPLOYS_PER_HOUR = 30;
/** Deploys one user scope may trigger per hour. */
export const EVENTS_USER_WORKER_DEPLOYS_PER_HOUR = 10;
/**
* Units one user may rent at once: ordinary workers, user scopes, and the
* events workers of apps they own. Only creating a unit is refused; holdings
* over the cap are kept.
*/
export const RENTED_UNITS_PER_USER = tiered(100, 16, 16);
/**
* Handlers one `publishAll` may carry. A build step publishes its whole set in
* one call, and the set is capped by what an app may hold anyway.
@@ -472,13 +448,11 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
// finish backs off the same way but is not the handler's failure, so it never
// counts toward that run.
// Wait before the first retry of a delivery a handler could not take, the
// longest wait, and the doubling between them. Shared with system jobs.
export {
deliveryBackoffMs,
EVENTS_RETRY_BASE_MS,
EVENTS_RETRY_MAX_MS,
} from '../../util/retryBackoff.js';
/** Wait before the first retry of a delivery a handler could not take. */
export const EVENTS_RETRY_BASE_MS = 2_000;
/** Longest wait between retries, however many have failed. */
export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
/**
* Handler failures in a row before a subscription is suspended. Counted per
@@ -494,6 +468,13 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
*/
export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
/** How long to hold a delivery whose handler has failed `attempts` times. */
export const deliveryBackoffMs = (attempts: number): number =>
Math.min(
EVENTS_RETRY_MAX_MS,
EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
);
// -- Suspended backlog -----------------------------------------------
//
// A suspended subscription stops metering but keeps what it is owed, and that
@@ -1,274 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The schedule routes are thin delegates: without `services.schedules` every
* one answers 503 `events_disabled`; with it, each hands its request over and
* answers what it returns.
*/
import type { Request, Response } from 'express';
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
it,
vi,
} from 'vitest';
import { makeActor, type Actor } from '../../core/actor.js';
import { isHttpError } from '../../core/http/HttpError.js';
import type {
ScheduleServiceContract,
ScheduleView,
} from '../../core/schedules.js';
import type { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import type { BroadcastController } from '../broadcast/BroadcastController.js';
import type { EventsController } from './EventsController.js';
let server: PuterServer;
let events: EventsController;
let broadcast: BroadcastController;
let actor: Actor;
let saved: unknown;
const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
const view = {
scheduleId: SCHEDULE_ID,
kind: 'recurring',
appUid: 'app-x',
} as ScheduleView;
const slot = () => server.services as unknown as { schedules?: unknown };
/** Put `schedules` in the slot, or leave the slot empty. */
const setSchedules = (schedules: unknown): void => {
if (schedules === undefined) delete slot().schedules;
else slot().schedules = schedules;
};
const fakeSchedules = () =>
({
onSubscriptionsChanged: vi.fn(async () => {}),
createFor: vi.fn(async () => view),
list: vi.fn(async () => ({
items: [view],
cursor: 'next',
total: 3,
})),
get: vi.fn(async () => view),
remove: vi.fn(async () => {}),
receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
prepareSendAt: vi.fn(),
createDeferredAppMessage: vi.fn(),
}) satisfies ScheduleServiceContract;
const req = (init: {
body?: unknown;
query?: Record<string, unknown>;
params?: Record<string, string>;
}): Request =>
({
actor,
body: init.body ?? {},
query: init.query ?? {},
params: init.params ?? {},
headers: {},
rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
}) as unknown as Request;
const res = () => {
const sent: { status: number; body: unknown } = {
status: 200,
body: undefined,
};
const response = {
json: vi.fn((body: unknown) => {
sent.body = body;
return response;
}),
status: vi.fn((code: number) => {
sent.status = code;
return response;
}),
setHeader: vi.fn(() => response),
};
return { response: response as unknown as Response, sent };
};
/** Each route, called as the router would. */
const routes: Array<[string, (response: Response) => Promise<void>]> = [
[
'POST /events/schedules',
(r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
],
[
'GET /events/schedules',
(r) => events.listSchedules(req({ query: { limit: '5' } }), r),
],
[
'GET /events/schedules/:scheduleId',
(r) =>
events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
],
[
'POST /events/schedules/remove',
(r) =>
events.removeSchedule(
req({ body: { scheduleId: SCHEDULE_ID } }),
r,
),
],
[
'POST /broadcast/events/schedules',
(r) =>
broadcast.schedules(
req({ body: { namespace: 'n', changes: [] } }),
r,
),
],
];
beforeAll(async () => {
server = await setupTestServer();
events = server.controllers.events as unknown as EventsController;
broadcast = server.controllers.broadcast as unknown as BroadcastController;
actor = makeActor({
user: { id: 1, uuid: 'u-1', username: 'someone' },
} as never);
saved = slot().schedules;
vi.spyOn(
server.services.broadcast,
'verifySignedRequest',
).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
});
afterEach(() => {
setSchedules(saved);
});
afterAll(async () => {
vi.restoreAllMocks();
await server?.shutdown();
});
describe('without services.schedules', () => {
it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
setSchedules(undefined);
const { response } = res();
let thrown: unknown;
try {
await call(response);
} catch (err) {
thrown = err;
}
expect(isHttpError(thrown)).toBe(true);
expect(thrown).toMatchObject({
statusCode: 503,
legacyCode: 'events_disabled',
// The designed answer, so it never pages.
noAlarm: true,
});
});
it('refuses an unsigned peer batch before saying schedules are off', async () => {
setSchedules(undefined);
vi.spyOn(
server.services.broadcast,
'verifySignedRequest',
).mockResolvedValueOnce({
ok: false,
status: 401,
message: 'bad signature',
} as never);
const { response, sent } = res();
await broadcast.schedules(
req({ body: { namespace: 'n', changes: [] } }),
response,
);
expect(sent.status).toBe(401);
});
});
describe('with services.schedules', () => {
it('creates through createFor and answers its view', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.createSchedule(req({ body: { every: 60_000 } }), response);
expect(schedules.createFor).toHaveBeenCalledWith(actor, {
every: 60_000,
});
expect(sent.body).toEqual(view);
});
it('lists with the limit capped, the cursor and includeTotal', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.listSchedules(
req({
query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
}),
response,
);
expect(schedules.list).toHaveBeenCalledWith(actor, {
limit: 200,
cursor: 'c',
includeTotal: true,
});
expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
});
it('reads one through get', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.getSchedule(
req({ params: { scheduleId: SCHEDULE_ID } }),
response,
);
expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
expect(sent.body).toEqual(view);
});
it('removes through remove and answers {}', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.removeSchedule(
req({ body: { scheduleId: SCHEDULE_ID } }),
response,
);
expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
expect(sent.body).toEqual({});
});
it('hands a signed peer batch to receivePeerChanges', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
const body = { namespace: 'n', changes: [] };
await broadcast.schedules(req({ body }), response);
expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
});
});
@@ -25,35 +25,9 @@
import MockRedis from 'ioredis-mock';
import { describe, expect, it } from 'vitest';
import type { fetch as undiciFetch } from 'undici';
import {
DispatcherInvokeTransport,
EVENTS_DEPLOYED_HEADER,
EVENTS_DISPATCH_ERROR_HEADER,
EVENTS_HANDLED_HEADER,
EventsWorkerInvokerClient,
type EventsInvokeCall,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import { INTERNAL_DEPLOY_TARGET } from '../../drivers/workers/WorkerDriver.js';
import {
EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_DEPLOYS_PER_HOUR,
} from './limits.js';
import {
EventsWorkerDeployer,
LocalEventsInvokeTransport,
type EventsDeployOptions,
type EventsDeployOutcome,
} from './workerDeploy.js';
import {
eventsWorkerScript,
userHandlerScript,
} from '../../services/events/workerRuntime.js';
import {
appHandlerScope,
hashContent,
type HandlerScope,
} from '../../stores/events/EventHandlerStore.js';
import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
import { EventsWorkerDeployer } from './workerDeploy.js';
import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
import { handlerSetHash } from '../../services/events/workerSource.js';
import type { IConfig } from '../../types.js';
@@ -72,22 +46,14 @@ interface Owner {
type Layers = ConstructorParameters<typeof EventsWorkerDeployer>[0];
/** Stub layers whose only real dependency is a fresh redis mock. */
const makeLayers = (
owners: Record<string, Owner | null>,
events: Record<string, unknown> = {},
) => {
const makeLayers = (owners: Record<string, Owner | null>) => {
const redis = new MockRedis.Cluster(['redis://localhost:7001']);
const created: string[] = [];
const targets: Array<Record<string, unknown>> = [];
/** Apps whose handler sources were read, once per read. */
const sourceReads: string[] = [];
const layers = {
config: {
events: {
workerRuntime: true,
internalSecret: 'sekret',
...events,
},
events: { workerRuntime: true, internalSecret: 'sekret' },
// `#namespace()` needs a local-worker shape to resolve without
// throwing — this stub never reaches a real Cloudflare or
// Miniflare deploy either way.
@@ -97,7 +63,7 @@ const makeLayers = (
stores: {
eventHandler: {
setForApp: async () => [{ name: 'h', sourceHash: SOURCE_HASH }],
allForApp: async ({ appUid }: { appUid: string }) => {
allForApp: async (appUid: string) => {
sourceReads.push(appUid);
return [
{
@@ -120,18 +86,6 @@ const makeLayers = (
},
},
user: {
getByUuid: async (uuid: string) => {
const owner = Object.values(owners).find(
(o) => `uuid-${o?.ownerUserId}` === uuid,
);
if (!owner) return null;
return {
id: owner.ownerUserId,
uuid,
username: `user-${owner.ownerUserId}`,
suspended: owner.suspended ?? false,
};
},
getById: async (id: number) => {
const owner = Object.values(owners).find(
(o) => o?.ownerUserId === id,
@@ -149,12 +103,8 @@ const makeLayers = (
services: {},
drivers: {
workers: {
create: async (args: {
workerName: string;
[INTERNAL_DEPLOY_TARGET]?: Record<string, unknown>;
}) => {
create: async (args: { workerName: string }) => {
created.push(args.workerName);
targets.push(args[INTERNAL_DEPLOY_TARGET] ?? {});
return { success: true, errors: [] };
},
},
@@ -163,7 +113,6 @@ const makeLayers = (
return {
layers: layers as unknown as Layers,
created,
targets,
redis,
sourceReads,
};
@@ -182,8 +131,8 @@ describe('EventsWorkerDeployer.ensure', () => {
const deployer = new EventsWorkerDeployer(layers);
const [a, b] = await Promise.all([
deployer.ensure(appHandlerScope('app-a'), SCRIPT),
deployer.ensure(appHandlerScope('app-b'), SCRIPT),
deployer.ensure('app-a', SCRIPT),
deployer.ensure('app-b', SCRIPT),
]);
expect(a).toBe('deployed');
expect(b).toBe('no-owner');
@@ -200,9 +149,7 @@ describe('EventsWorkerDeployer.ensure', () => {
String(EVENTS_WORKER_DEPLOYS_PER_HOUR),
);
expect(await deployer.ensure(appHandlerScope('app-c'), SCRIPT)).toBe(
'throttled',
);
expect(await deployer.ensure('app-c', SCRIPT)).toBe('throttled');
expect(sourceReads).toEqual([]);
expect(created).toEqual([]);
});
@@ -217,680 +164,15 @@ describe('EventsWorkerDeployer.ensure', () => {
'',
);
expect(await deployer.ensure(appHandlerScope('app-e'), stale)).toBe(
'stale',
);
expect(await deployer.ensure('app-e', stale)).toBe('stale');
expect(sourceReads).toEqual([]);
const hour = new Date().toISOString().slice(0, 13);
expect(await redis.get(`ev:deploys:{app-e}:${hour}`)).toBeNull();
});
it('passes events.workerCpuMs to the deploy target', async () => {
const { layers, targets } = makeLayers(
{ 'app-f': { ownerUserId: 6 } },
{ workerCpuMs: 50 },
);
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.ensure(appHandlerScope('app-f'), SCRIPT)).toBe(
'deployed',
);
expect(targets[0]!.cpuMs).toBe(50);
});
it('passes no cpuMs when events.workerCpuMs is unset', async () => {
const { layers, targets } = makeLayers({ 'app-g': { ownerUserId: 7 } });
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.ensure(appHandlerScope('app-g'), SCRIPT)).toBe(
'deployed',
);
expect(targets[0]!.cpuMs).toBeUndefined();
});
it('deploys normally under the cap', async () => {
const { layers } = makeLayers({ 'app-d': { ownerUserId: 4 } });
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.ensure(appHandlerScope('app-d'), SCRIPT)).toBe(
'deployed',
);
});
});
/** A server's schedules with a deploy gate that records and can refuse. */
const withGate = (
layers: Layers,
refuse: (capped: boolean) => boolean = () => false,
) => {
const gated: Array<{ script: string; capped: boolean }> = [];
const gate = {
gateDeploy: async (
script: string,
options: { capped: boolean },
deploy: () => Promise<EventsDeployOutcome>,
): Promise<EventsDeployOutcome> => {
gated.push({ script, capped: options.capped });
return refuse(options.capped) ? 'busy' : deploy();
},
};
const deployer = new EventsWorkerDeployer({
...layers,
services: { ...layers.services, schedules: gate },
} as unknown as Layers);
return { deployer, gated };
};
describe('the schedules` deploy gate', () => {
it('is skipped without schedules, or with schedules that have no gate', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
await expect(
new EventsWorkerDeployer(layers).ensure(
appHandlerScope('app-x'),
SCRIPT,
{
lane: 'schedule',
},
),
).resolves.toBe('deployed');
const gateless = new EventsWorkerDeployer({
...layers,
services: { ...layers.services, schedules: {} },
} as unknown as Layers);
await expect(
gateless.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
}),
).resolves.toBe('deployed');
expect(created).toEqual([SCRIPT, SCRIPT]);
});
it('takes every deploy, capped only for a schedule run', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer, gated } = withGate(layers);
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
).resolves.toBe('deployed');
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
}),
).resolves.toBe('deployed');
expect(gated).toEqual([
{ script: SCRIPT, capped: false },
{ script: SCRIPT, capped: true },
]);
expect(created).toEqual([SCRIPT, SCRIPT]);
});
it('answers a refused schedule run busy, and a delivery still deploys', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer } = withGate(layers, (capped) => capped);
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
}),
).resolves.toBe('busy');
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
).resolves.toBe('deployed');
expect(created).toEqual([SCRIPT]);
});
it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer, gated } = withGate(layers, (capped) => capped);
const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
});
const deliveries = [
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
];
await expect(scheduled).resolves.toBe('busy');
await expect(Promise.all(deliveries)).resolves.toEqual([
'deployed',
'deployed',
]);
expect(created).toEqual([SCRIPT]);
expect(gated.map((g) => g.capped)).toEqual([true, false]);
});
it('lets a schedule run join a delivery`s deploy in flight', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer, gated } = withGate(layers, (capped) => capped);
const delivery = deployer.ensure(appHandlerScope('app-x'), SCRIPT);
const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
});
await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
'deployed',
'deployed',
]);
expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
expect(created).toEqual([SCRIPT]);
});
});
describe('LocalEventsInvokeTransport', () => {
it('renders each request as it goes out, after a cold deploy', async () => {
const bodies: string[] = [];
let resident = false;
const transport = new LocalEventsInvokeTransport(
{
localworkerservice: {
dispatchEventsWorker: async (request: Request) => {
bodies.push(await request.text());
if (!resident) return null;
return new Response(null, {
status: 200,
headers: { 'x-puter-events-handled': '1' },
});
},
},
} as never,
{
ensure: async () => {
resident = true;
return 'deployed';
},
scopeFor: async (appUid: string) => appHandlerScope(appUid),
} as unknown as EventsWorkerDeployer,
);
let renders = 0;
const call: EventsInvokeCall = {
script: SCRIPT,
appUid: 'app-x',
key: 'k',
body: '{"n":0}',
timeoutMs: 5000,
render: () => `{"n":${++renders}}`,
};
await expect(transport.send(call)).resolves.toMatchObject({
status: 200,
});
expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
});
});
describe('deploy-on-miss of an evicted script', () => {
it('redeploys the evicted script, then resends the run with the deployed marker', async () => {
const { layers, created } = makeLayers({ 'app-m': { ownerUserId: 8 } });
// The namespace only has the script once the deploy lands: before
// that the dispatcher answers a miss.
let resident = false;
layers.drivers.workers.create = (async (args: {
workerName: string;
}) => {
created.push(args.workerName);
resident = true;
return { success: true, errors: [] };
}) as never;
const deployer = new EventsWorkerDeployer(layers);
const sent: boolean[] = [];
const fetchImpl = (async (_url: string, init?: RequestInit) => {
const headers = (init?.headers ?? {}) as Record<string, string>;
sent.push(headers[EVENTS_DEPLOYED_HEADER] === '1');
if (resident)
return new Response(null, {
status: 200,
headers: { [EVENTS_HANDLED_HEADER]: '1' },
});
return new Response(null, {
status: 404,
headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'missing' },
});
}) as unknown as typeof undiciFetch;
const client = new EventsWorkerInvokerClient({
events: { workerRuntime: true },
} as IConfig);
client.setTransport(
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl,
}),
);
client.setMissHandler((appUid, script, options) =>
deployer.ensure(appHandlerScope(appUid), script, options),
);
const result = await client.invoke({
script: SCRIPT,
appUid: 'app-m',
handler: 'h',
token: 't',
key: 'k',
event: {},
ctx: {},
});
expect(result).toEqual({ outcome: 'settled', status: 200 });
expect(created).toEqual([SCRIPT]);
// One miss, then the resend carrying the deployed marker.
expect(sent).toEqual([false, true]);
});
});
// The redis mock is shared across tests, so each one uses its own app and code.
let fresh = 0;
const nextApp = () => `app-u${++fresh}`;
const userScope = (holder: number, appUid: string) => ({ appUid, holder });
const sourceN = (label: string) =>
`async () => { await ${label}${++fresh}(); }`;
const scriptOf = (name: string, source: string) =>
userHandlerScript({ name, sourceHash: hashContent(source) }, '');
type Sets = Record<number, Array<{ name: string; source: string }>>;
/** Layers whose handler sets are per holder (0 is the app's own). */
const withSets = (layers: Layers, sets: Sets) => {
const handlersOf = (scope: { appUid: string; holder: number }) =>
(sets[scope.holder] ?? []).map((h) => ({
appUid: scope.appUid,
name: h.name,
source: h.source,
sourceHash: hashContent(h.source),
createdAt: 0,
updatedAt: 0,
}));
const store = layers.stores.eventHandler as unknown as Record<
string,
unknown
>;
store.setForApp = async (scope: { appUid: string; holder: number }) =>
handlersOf(scope).map(({ name, sourceHash }) => ({ name, sourceHash }));
store.allForApp = async (scope: { appUid: string; holder: number }) =>
handlersOf(scope);
};
const holders = (...ids: number[]) =>
Object.fromEntries(
ids.map((id) => [`holder-${id}`, { ownerUserId: id }]),
) as Record<string, Owner>;
describe('EventsWorkerDeployer in a user scope', () => {
it('deploys the one handler whose script was asked for, as the holder', async () => {
const app = nextApp();
const a = sourceN('a');
const b = sourceN('b');
const { layers, created, targets } = makeLayers(holders(10));
withSets(layers, {
10: [
{ name: 'a', source: a },
{ name: 'b', source: b },
],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('b', b);
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script]);
const target = targets[0]!;
expect(target.tags).toEqual([`events-app:${app}`, 'user:uuid-10']);
// One handler's script: the other one's code is not in it.
expect(target.source).toContain(b);
expect(target.source).not.toContain(a);
});
it('answers a script no handler of the scope matches as stale, before the throttle', async () => {
const app = nextApp();
const { layers, redis, created } = makeLayers(holders(10));
withSets(layers, { 10: [{ name: 'a', source: sourceN('a') }] });
const deployer = new EventsWorkerDeployer(layers);
expect(
await deployer.ensure(
userScope(10, app),
scriptOf('b', sourceN('b')),
),
).toBe('stale');
expect(created).toEqual([]);
const hour = new Date().toISOString().slice(0, 13);
expect(await redis.get(`ev:deploys:{${app}}:10:${hour}`)).toBeNull();
});
it('answers a scope with no handlers as no-handlers', async () => {
const { layers } = makeLayers(holders(10));
withSets(layers, {});
const deployer = new EventsWorkerDeployer(layers);
expect(
await deployer.ensure(
userScope(10, nextApp()),
scriptOf('a', sourceN('a')),
),
).toBe('no-handlers');
});
it('does not deploy for a holder that is suspended or gone', async () => {
const source = sourceN('a');
const { layers, created } = makeLayers({
'holder-10': { ownerUserId: 10, suspended: true },
});
withSets(layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
const app = nextApp();
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'no-owner',
);
expect(await deployer.ensure(userScope(11, app), script)).toBe(
'no-owner',
);
expect(created).toEqual([]);
});
it('gives each scope its own hourly cap, apart from the app scope', async () => {
const app = nextApp();
const source = sourceN('a');
const { layers, created } = makeLayers({
...holders(10, 11),
[app]: { ownerUserId: 12 },
});
withSets(layers, {
0: [{ name: 'a', source }],
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
for (let i = 0; i < EVENTS_USER_WORKER_DEPLOYS_PER_HOUR; i++)
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'deployed',
);
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'throttled',
);
expect(created).toHaveLength(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR);
// Another holder of the same app, and the app's own scope, still go.
expect(await deployer.ensure(userScope(11, app), script)).toBe(
'deployed',
);
expect(await deployer.ensure(appHandlerScope(app), script)).toBe(
'deployed',
);
});
it('collapses concurrent deploys of one script across holders into one upload', async () => {
const app = nextApp();
const source = sourceN('a');
const { layers, created } = makeLayers(holders(10, 11));
withSets(layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
await Promise.all([
deployer.ensure(userScope(10, app), script),
deployer.ensure(userScope(10, app), script),
deployer.ensure(userScope(11, app), script),
deployer.ensure(userScope(11, nextApp()), script),
]);
expect(created).toHaveLength(1);
});
it('names a script for identical code the same under any holder and app', async () => {
const source = sourceN('a');
const { layers, created } = makeLayers(holders(10, 11));
withSets(layers, {
10: [{ name: '@x', source }],
11: [{ name: '@x', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('@x', source);
expect(await deployer.ensure(userScope(10, nextApp()), script)).toBe(
'deployed',
);
expect(await deployer.ensure(userScope(11, nextApp()), script)).toBe(
'deployed',
);
expect(created).toEqual([script, script]);
});
});
describe('the deployed marker', () => {
const setup = () => {
const source = sourceN('a');
const made = makeLayers(holders(10, 11));
withSets(made.layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
return {
...made,
deployer: new EventsWorkerDeployer(made.layers),
script: scriptOf('a', source),
app: nextApp(),
};
};
it('is left by a user-scope deploy with a ttl under a day', async () => {
const { deployer, redis, script, app } = setup();
await deployer.ensure(userScope(10, app), script);
const ttl = await redis.ttl(`ev:deployed:${script}`);
expect(ttl).toBeGreaterThan(0);
expect(ttl).toBeLessThan(24 * 60 * 60);
});
it('lets an eager deploy skip the upload another holder already made', async () => {
const { deployer, created, script, app } = setup();
expect(await deployer.deployEagerly(userScope(10, app), script)).toBe(
'deployed',
);
expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script]);
});
it('spends no throttle on a skipped eager deploy', async () => {
const { deployer, redis, script, app } = setup();
await deployer.deployEagerly(userScope(10, app), script);
await deployer.deployEagerly(userScope(11, app), script);
const hour = new Date().toISOString().slice(0, 13);
expect(await redis.get(`ev:deploys:{${app}}:11:${hour}`)).toBeNull();
});
it('never skips a deploy-on-miss', async () => {
const { deployer, created, script, app } = setup();
await deployer.deployEagerly(userScope(10, app), script);
expect(await deployer.ensure(userScope(11, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script, script]);
});
it('is not left by an app-scope deploy', async () => {
const app = nextApp();
const source = sourceN('a');
const { layers, redis, created } = makeLayers({
[app]: { ownerUserId: 4 },
});
withSets(layers, { 0: [{ name: 'a', source }] });
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
await deployer.ensure(appHandlerScope(app), script);
expect(created).toEqual([script]);
expect(await redis.get(`ev:deployed:${script}`)).toBeNull();
});
it('still deploys eagerly when the marker cannot be read', async () => {
const { deployer, redis, created, script, app } = setup();
await deployer.deployEagerly(userScope(10, app), script);
const get = redis.get.bind(redis);
redis.get = (async (key: string) => {
if (key.startsWith('ev:deployed:')) throw new Error('down');
return get(key);
}) as never;
expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script, script]);
});
});
describe('scopeFor', () => {
it('reads no holder as the app scope, and resolves a uuid to the user scope', async () => {
const { layers } = makeLayers(holders(10));
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.scopeFor('app-u')).toEqual({
appUid: 'app-u',
holder: 0,
});
expect(await deployer.scopeFor('app-u', '')).toEqual({
appUid: 'app-u',
holder: 0,
});
expect(await deployer.scopeFor('app-u', 'uuid-10')).toEqual({
appUid: 'app-u',
holder: 10,
});
expect(await deployer.scopeFor('app-u', 'uuid-99')).toBeNull();
});
});
describe('LocalEventsInvokeTransport in a user scope', () => {
it('deploys the holder`s script on a miss and refuses an unknown holder', async () => {
const source = sourceN('a');
const { layers, created } = makeLayers(holders(10));
withSets(layers, { 10: [{ name: 'a', source }] });
const deployer = new EventsWorkerDeployer(layers);
let resident = false;
const transport = new LocalEventsInvokeTransport(
{
localworkerservice: {
dispatchEventsWorker: async () => {
if (!resident) return null;
return new Response(null, {
status: 200,
headers: { 'x-puter-events-handled': '1' },
});
},
},
} as never,
{
ensure: async (
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
) => {
const outcome = await deployer.ensure(
scope,
script,
options,
);
resident = outcome === 'deployed';
return outcome;
},
scopeFor: (appUid: string, holder?: string) =>
deployer.scopeFor(appUid, holder),
} as unknown as EventsWorkerDeployer,
);
const script = scriptOf('a', source);
const call: EventsInvokeCall = {
script,
appUid: nextApp(),
holder: 'uuid-10',
key: 'k',
body: '{}',
timeoutMs: 5000,
};
await expect(transport.send(call)).resolves.toMatchObject({
status: 200,
});
expect(created).toEqual([script]);
resident = false;
await expect(
transport.send({ ...call, holder: 'uuid-99' }),
).resolves.toMatchObject({ status: null, error: 'deploy: no-holder' });
expect(created).toEqual([script]);
});
});
describe('concurrent eager deploys of one script', () => {
const setup = () => {
const source = sourceN('a');
const made = makeLayers(holders(10, 11));
withSets(made.layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
return {
...made,
script: scriptOf('a', source),
app: nextApp(),
};
};
it('upload once when two holders publish the same new code together', async () => {
const { layers, created, script, app } = setup();
const deployer = new EventsWorkerDeployer(layers);
const outcomes = await Promise.all([
deployer.deployEagerly(userScope(10, app), script),
deployer.deployEagerly(userScope(11, app), script),
]);
expect(outcomes).toEqual(['deployed', 'deployed']);
expect(created).toEqual([script]);
});
it('skip the upload when the marker appears while they wait on the gate', async () => {
const { layers, redis, created, script, app } = setup();
// Another node's deploy of the script finishes before the gate opens.
const gated = new EventsWorkerDeployer({
...layers,
services: {
schedules: {
gateDeploy: async (
_script: string,
_options: { capped: boolean },
deploy: () => Promise<EventsDeployOutcome>,
) => {
await redis.set(`ev:deployed:${script}`, '1');
return deploy();
},
},
},
} as unknown as Layers);
expect(
await gated.ensure(userScope(10, app), script, { eager: true }),
).toBe('deployed');
expect(created).toEqual([]);
// A miss still uploads, whatever the marker says.
expect(await gated.ensure(userScope(11, app), script)).toBe('deployed');
expect(created).toEqual([script]);
});
it('make a miss that joins an eager deploy deploy after it', async () => {
const { layers, redis, created, script, app } = setup();
const deployer = new EventsWorkerDeployer(layers);
await redis.set(`ev:deployed:${script}`, '1');
const eager = deployer.ensure(userScope(10, app), script, {
eager: true,
});
const miss = deployer.ensure(userScope(11, app), script);
expect(await eager).toBe('deployed');
expect(await miss).toBe('deployed');
// The eager one skipped on the marker; the miss uploaded.
expect(created).toEqual([script]);
expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
});
});
+44 -282
View File
@@ -29,7 +29,6 @@ import {
import type {
EventsInvokeAnswer,
EventsInvokeCall,
InvokeLane,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import {
EVENTS_INVOKE_PATH,
@@ -37,89 +36,47 @@ import {
failedAnswer,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import type { puterServices } from '../../services/index.js';
import type { IExtensionServiceInstances } from '../../services/types.js';
import {
eventsInvokeKey,
eventsWorkerScope,
eventsWorkerScript,
userHandlerScript,
} from '../../services/events/workerRuntime.js';
import {
generateEventsWorkerSource,
handlerSetHash,
} from '../../services/events/workerSource.js';
import {
appHandlerScope,
type HandlerScope,
} from '../../stores/events/EventHandlerStore.js';
import type { puterStores } from '../../stores/index.js';
import type { IConfig, LayerInstances } from '../../types.js';
import {
EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_DEPLOYS_PER_HOUR,
} from './limits.js';
import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
/**
* Deploys an app's events worker from its published handlers.
*
* Deployed the first time a delivery needs it and again on eviction — a publish
* only writes rows (a user-scope publish also deploys eagerly). An app's scope
* is one script for the whole set; a user scope is one script per handler,
* deployed as the holder. Scripts carry no worker token and no `subdomains`
* row: not one of the owner's workers, not addressable from the internet, not
* listed or deleted as if it were. A republished set leaves the old script
* behind rather than deleting it; redeploying a still-live one is always safe.
* only writes rows. Scripts carry no worker token and no `subdomains` row: not
* one of the owner's workers, not addressable from the internet, not listed or
* deleted as if it were. A republished set leaves the old script behind rather
* than deleting it; redeploying a still-live one is always safe.
*/
/** Why a deploy could not happen, for the callers that answer differently. */
export type EventsDeployOutcome =
| 'deployed'
| 'stale'
| 'no-handlers'
| 'no-owner'
| 'throttled'
| 'failed'
| 'busy';
/** Whose call asked for the deploy: a schedule run's is held to a share. */
export interface EventsDeployOptions {
lane?: InvokeLane;
/**
* A user-scope publish's deploy, which skips the upload when the script's
* deployed marker is already set. A deploy-on-miss is never eager.
*/
eager?: boolean;
}
'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
interface DeployLayers {
config: IConfig;
clients: LayerInstances<typeof puterClients>;
stores: LayerInstances<typeof puterStores>;
services: LayerInstances<typeof puterServices> &
Pick<IExtensionServiceInstances, 'schedules'>;
services: LayerInstances<typeof puterServices>;
drivers: LayerInstances<typeof puterDrivers>;
}
/**
* `ev:deploys:{<appUid>}[:<holder>]:<hour>` bucket TTL — comfortably past the
* hour it counts.
* `ev:deploys:{<appUid>}:<hour>` bucket TTL — comfortably past the hour it
* counts.
*/
const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
/** How long `ev:deployed:<script>` says a script was uploaded. Under a day. */
const DEPLOYED_MARKER_TTL_SECONDS = 20 * 60 * 60;
const deployedMarkerKey = (script: string): string => `ev:deployed:${script}`;
const isUserScope = (scope: HandlerScope): boolean => scope.holder !== 0;
interface InFlightDeploy {
/** A schedule run's, which the region's deploy slots may refuse. */
capped: boolean;
eager: boolean;
promise: Promise<EventsDeployOutcome>;
}
interface DeployResult {
success?: boolean;
errors?: unknown[];
@@ -128,7 +85,7 @@ interface DeployResult {
export class EventsWorkerDeployer {
readonly #layers: DeployLayers;
/** Collapses the burst of misses one cold app produces into one deploy. */
readonly #inFlight = new Map<string, InFlightDeploy>();
readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
constructor(layers: DeployLayers) {
this.#layers = layers;
@@ -143,188 +100,44 @@ export class EventsWorkerDeployer {
}
/**
* Deploy `script`, which must be what the scope's handlers currently hash
* to: the whole set for an app, one handler for a user scope. A request for
* anything else is `stale` — the handlers moved on, and the next delivery
* resolves the new script rather than this one being revived. Always
* uploads; a deploy-on-miss means whatever said it was there is wrong.
* Deploy `script`, which must be what the app's handlers currently hash to.
* A request for anything else is `stale` — the set moved on, and the next
* delivery resolves the new script rather than this one being revived.
*/
ensure(
scope: HandlerScope,
script: string,
options: EventsDeployOptions = {},
): Promise<EventsDeployOutcome> {
// A user scope's script is the same bytes whoever holds it, so
// concurrent deploys of it from any holder or app are one.
const key = isUserScope(scope)
? `user|${script}`
: `${scope.appUid}|${scope.holder}|${script}`;
const capped = options.lane === 'schedule';
ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
const key = `${appUid}|${script}`;
const existing = this.#inFlight.get(key);
if (!existing) return this.#start(key, scope, script, options);
// An eager deploy may have skipped the upload, which a miss must not
// take for done.
if (existing.eager && !options.eager)
return this.#afterEager(
existing.promise,
key,
scope,
script,
options,
);
if (capped || !existing.capped) return existing.promise;
return this.#afterCapped(existing.promise, key, scope, script, options);
}
/**
* {@link ensure} for the deploy a user-scope publish starts, which skips the
* upload when another holder's deploy of the same script left its marker.
* Only this path reads the marker.
*/
async deployEagerly(
scope: HandlerScope,
script: string,
): Promise<EventsDeployOutcome> {
if (await this.#markedDeployed(script)) return 'deployed';
return this.ensure(scope, script, { eager: true });
}
/**
* The scope a script request names: the app's own, or the user scope of
* `holder` (a user uuid). Null for a holder that does not exist.
*/
async scopeFor(
appUid: string,
holder?: string | null,
): Promise<HandlerScope | null> {
if (!holder) return appHandlerScope(appUid);
const user = await this.#layers.stores.user.getByUuid(holder);
if (!user?.id) return null;
return { appUid, holder: Number(user.id) };
}
#start(
key: string,
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
const entry: InFlightDeploy = {
capped: options.lane === 'schedule',
eager: options.eager === true,
promise: this.#gated(scope, script, options).finally(() => {
if (this.#inFlight.get(key) === entry)
this.#inFlight.delete(key);
}),
};
this.#inFlight.set(key, entry);
return entry.promise;
}
/** A miss that found an eager deploy in flight deploys after it. */
async #afterEager(
eager: Promise<EventsDeployOutcome>,
key: string,
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
await eager;
const current = this.#inFlight.get(key);
if (current && !current.eager) return current.promise;
return this.#start(key, scope, script, options);
}
/**
* A delivery that joined a schedule run's deploy: `busy` means no deploy
* started, and a delivery is never refused a slot, so it asks again.
*/
async #afterCapped(
capped: Promise<EventsDeployOutcome>,
key: string,
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
const outcome = await capped;
if (outcome !== 'busy') return outcome;
const current = this.#inFlight.get(key);
if (current && !current.capped) return current.promise;
return this.#start(key, scope, script, options);
}
/**
* Every deploy goes through the schedules' deploy gate when a server has
* one: a per-script lock, so a burst of runs of an evicted script deploys
* it once, and a region-wide cap that a schedule run's deploy (`capped`)
* may be refused with `busy`. A delivery's deploy is never refused.
*/
#gated(
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
const deploy = () =>
this.#deploy(scope, script, options.eager === true);
return (
this.#layers.services.schedules?.gateDeploy?.(
script,
{ capped: options.lane === 'schedule' },
deploy,
) ?? deploy()
if (existing) return existing;
const started = this.#deploy(appUid, script).finally(() =>
this.#inFlight.delete(key),
);
this.#inFlight.set(key, started);
return started;
}
async #deploy(
handlerScope: HandlerScope,
appUid: string,
script: string,
eager: boolean,
): Promise<EventsDeployOutcome> {
const { appUid } = handlerScope;
const { stores, drivers } = this.#layers;
const backend = eventsWorkerScope(this.#layers.config);
const userScope = isUserScope(handlerScope);
const scope = eventsWorkerScope(this.#layers.config);
// Names and hashes are enough to tell a stale request, so the set's
// sources are read and built only once the throttle has let it through.
const set = await stores.eventHandler.setForApp(handlerScope);
const set = await stores.eventHandler.setForApp(appUid);
if (set.length === 0) return 'no-handlers';
// Two apps that published byte-identical sets share one script, which
// is sound: the source is the same, and everything per-app arrives with
// the invocation. Whoever deploys first owns it. A user scope's script
// is one handler's, so it is shared the same way across holders.
const wanted = userScope
? set.find((entry) => userHandlerScript(entry, backend) === script)
: undefined;
if (
userScope
? !wanted
: eventsWorkerScript(handlerSetHash(set), backend) !== script
)
// the invocation. Whoever deploys first owns it.
if (eventsWorkerScript(handlerSetHash(set), scope) !== script)
return 'stale';
// Asked again inside the gate, where another deploy of this script may
// have just finished.
if (eager && userScope && (await this.#markedDeployed(script)))
return 'deployed';
if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
if (await this.#tooManyDeploysThisHour(handlerScope))
return 'throttled';
const all = await stores.eventHandler.allForApp(handlerScope);
const handlers = wanted
? all.filter(
(handler) =>
handler.name === wanted.name &&
handler.sourceHash === wanted.sourceHash,
)
: all;
const handlers = await stores.eventHandler.allForApp(appUid);
const generated = generateEventsWorkerSource(handlers);
// The set can move between the two reads.
if (
handlers.length === 0 ||
eventsWorkerScript(generated.setHash, backend) !== script
)
if (eventsWorkerScript(generated.setHash, scope) !== script)
return 'stale';
if (generated.tooLarge) {
console.error(
@@ -338,7 +151,7 @@ export class EventsWorkerDeployer {
generated.broken.join(', '),
);
const owner = await this.#owner(handlerScope);
const owner = await this.#owner(appUid);
if (!owner) return 'no-owner';
const secret = this.#layers.config.events?.internalSecret;
@@ -367,7 +180,6 @@ export class EventsWorkerDeployer {
runtime: 'events',
source: generated.source,
omitOwnerToken: true,
cpuMs: this.#layers.config.events?.workerCpuMs,
// What the script compares an invocation against.
// Derived from the script name, so a redeploy of
// the same set binds the same key.
@@ -398,74 +210,34 @@ export class EventsWorkerDeployer {
);
return 'failed';
}
if (userScope) await this.#markDeployed(script);
return 'deployed';
}
async #markedDeployed(script: string): Promise<boolean> {
try {
return !!(await this.#layers.clients.redis.get(
deployedMarkerKey(script),
));
} catch (err) {
console.warn('[events] deployed marker unreadable', err);
return false;
}
}
async #markDeployed(script: string): Promise<void> {
try {
await this.#layers.clients.redis.set(
deployedMarkerKey(script),
'1',
'EX',
DEPLOYED_MARKER_TTL_SECONDS,
);
} catch (err) {
console.warn('[events] could not mark a script deployed', err);
}
}
/**
* Whether this scope has already deployed its hourly cap this hour. A
* flapping set (publish, remove, republish) provisions upstream on every
* miss, and this is the ceiling on that loop. An app's counter is per app;
* a user scope has its own, so one user cannot spend another's.
* Whether this app has already deployed `EVENTS_WORKER_DEPLOYS_PER_HOUR`
* times this hour. A flapping set (publish, remove, republish) provisions
* upstream on every miss, and this is the ceiling on that loop.
*/
async #tooManyDeploysThisHour(scope: HandlerScope): Promise<boolean> {
async #tooManyDeploysThisHour(appUid: string): Promise<boolean> {
const redis = this.#layers.clients.redis;
const hour = new Date().toISOString().slice(0, 13);
const userScope = isUserScope(scope);
const key = userScope
? `ev:deploys:{${scope.appUid}}:${scope.holder}:${hour}`
: `ev:deploys:{${scope.appUid}}:${hour}`;
const key = `ev:deploys:{${appUid}}:${hour}`;
const count = await redis.incr(key);
if (count === 1) await redis.expire(key, DEPLOY_THROTTLE_TTL_SECONDS);
return (
count >
(userScope
? EVENTS_USER_WORKER_DEPLOYS_PER_HOUR
: EVENTS_WORKER_DEPLOYS_PER_HOUR)
);
return count > EVENTS_WORKER_DEPLOYS_PER_HOUR;
}
/**
* The user a scope's deploy runs as, as an actor: the app's owner, or the
* holder of a user scope.
*/
/** The app owner, as an actor the deploy can run as. */
async #owner(
scope: HandlerScope,
appUid: string,
): Promise<
| (Actor & { user: { id: number; uuid: string; username: string } })
| null
> {
const { stores } = this.#layers;
let ownerUserId = scope.holder;
if (!isUserScope(scope)) {
const app = await stores.app.getByUid(scope.appUid);
ownerUserId = Number(app?.owner_user_id);
if (!app || !Number.isFinite(ownerUserId)) return null;
}
const app = await stores.app.getByUid(appUid);
const ownerUserId = Number(app?.owner_user_id);
if (!app || !Number.isFinite(ownerUserId)) return null;
const user = await stores.user.getById(ownerUserId);
if (!user?.id || !user.uuid || !user.username || user.suspended)
@@ -527,8 +299,6 @@ export class LocalEventsInvokeTransport {
}
async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
// Built per attempt, so a send time stamped into the body is taken
// after a cold deploy rather than before it.
const request = (): Request =>
new Request(
`http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -538,7 +308,7 @@ export class LocalEventsInvokeTransport {
'content-type': 'application/json',
'x-puter-events-key': call.key,
},
body: call.render ? call.render() : call.body,
body: call.body,
},
);
@@ -546,20 +316,12 @@ export class LocalEventsInvokeTransport {
const resident = await this.#dispatch(request(), call.timeoutMs);
if (resident !== null) return resident;
const scope = await this.#deployer.scopeFor(
const outcome = await this.#deployer.ensure(
call.appUid,
call.holder,
call.script,
);
if (!scope) return { status: null, error: 'deploy: no-holder' };
const outcome = await this.#deployer.ensure(scope, call.script, {
lane: call.lane,
});
if (outcome !== 'deployed')
return {
status: null,
error: `deploy: ${outcome}`,
...(outcome === 'busy' ? { deployBusy: true } : {}),
};
return { status: null, error: `deploy: ${outcome}` };
const retried = await this.#dispatch(request(), call.timeoutMs);
if (retried === null)
@@ -78,13 +78,11 @@ interface CapturedResponse {
statusCode: number;
body: unknown;
contentType?: string;
location?: string;
}
const makeReq = (init: {
params?: Record<string, unknown>;
path?: string;
originalUrl?: string;
actor?: Actor;
hostname?: string;
protocol?: string;
@@ -95,7 +93,6 @@ const makeReq = (init: {
headers: {},
params: init.params ?? {},
path: init.path ?? '/',
originalUrl: init.originalUrl ?? init.path ?? '/',
hostname: init.hostname ?? 'test.local',
protocol: init.protocol ?? 'http',
actor: init.actor,
@@ -133,11 +130,6 @@ const makeRes = () => {
captured.contentType = value;
return res;
}),
redirect: vi.fn((code: number, location: string) => {
captured.statusCode = code;
captured.location = location;
return res;
}),
};
return { res: res as unknown as Response, captured };
};
@@ -221,12 +213,14 @@ describe('HomepageController shell routes', () => {
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
});
it('serves the shell on /settings, /dashboard, /action, /@:username', async () => {
it('serves the shell on /settings, /dashboard, /desktop, /action, /@:username', async () => {
for (const path of [
'/settings',
'/settings/*splat',
'/dashboard',
'/dashboard/',
'/desktop',
'/desktop/',
'/action/*splat',
'/@:username',
]) {
@@ -236,18 +230,6 @@ describe('HomepageController shell routes', () => {
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
}
});
it('serves the shell on a path inside a public folder, /@:username/*splat', async () => {
const { res, captured } = makeRes();
await callRoute(
'get',
'/@:username/*splat',
makeReq({ path: '/@alice/photos/cat.png' }),
res,
);
expect(captured.statusCode).toBe(200);
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
});
});
// ── /app/:name ──────────────────────────────────────────────────────
@@ -377,6 +359,49 @@ describe('HomepageController GET /app/:name', () => {
expect(html).not.toContain('owner_user_id');
});
it('serves the same app shell under /desktop/app/:name', async () => {
const { userId } = await makeUser();
const name = `desk-${Math.random().toString(36).slice(2, 10)}`;
await server.stores.app.create(
{
name,
title: 'Desktop App',
description: 'opens on the desktop instead of the dashboard',
index_url: `https://example.com/${name}/`,
approved_for_listing: 1,
},
{ ownerUserId: userId },
);
const { res, captured } = makeRes();
await callRoute(
'get',
'/desktop/app/:name',
makeReq({ params: { name }, path: `/desktop/app/${name}` }),
res,
);
expect(captured.statusCode).toBe(200);
const html = String(captured.body);
expect(html).toMatch(/<!DOCTYPE html>/i);
expect(html).toContain('Desktop App');
});
it('returns 404 under /desktop/app/:name when the app is unknown', async () => {
const { res, captured } = makeRes();
await callRoute(
'get',
'/desktop/app/:name',
makeReq({
params: { name: 'no-such-app' },
path: '/desktop/app/no-such-app',
}),
res,
);
expect(captured.statusCode).toBe(404);
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
});
it('omits index_url even for a public app — the shell is not the launch authority', async () => {
const { userId } = await makeUser();
const name = `pub-${Math.random().toString(36).slice(2, 10)}`;
@@ -408,8 +433,8 @@ describe('HomepageController GET /app/:name', () => {
// -- app landing rate limit --
describe('HomepageController app landing rate limit', () => {
it('rate-limits /app/:name per network', () => {
for (const path of ['/app/:name']) {
it('rate-limits /app/:name and /desktop/app/:name per network', () => {
for (const path of ['/app/:name', '/desktop/app/:name']) {
const route = router.routes.find(
(r) => r.method === 'get' && r.path === path,
);
@@ -427,7 +452,7 @@ describe('HomepageController app landing rate limit', () => {
// ── /show/* ─────────────────────────────────────────────────────────
describe('HomepageController GET /show/*splat', () => {
it('serves the plain shell; the GUI opens the path itself', async () => {
it('emits a launch_app explorer call with the post-/show path', async () => {
const { res, captured } = makeRes();
await callRoute(
'get',
@@ -435,52 +460,13 @@ describe('HomepageController GET /show/*splat', () => {
makeReq({ path: '/show/alice/Documents' }),
res,
);
// The launch payload is JSON-serialized into the rendered HTML.
// We assert the rendered shell included the explorer launch hint
// pointing at the expected (slashed) path.
const html = String(captured.body);
expect(html).toMatch(/<!DOCTYPE html>/i);
// No boot-time launch hook: the one it used to carry named a GUI
// global that no longer exists, which stopped the GUI booting.
expect(html).not.toContain('launch_app');
});
});
// ── /desktop ────────────────────────────────────────────────────────
describe('HomepageController GET /desktop', () => {
it('redirects /desktop to the root, keeping the query', async () => {
for (const path of ['/desktop', '/desktop/']) {
const { res, captured } = makeRes();
await callRoute('get', path, makeReq({ path }), res);
expect(captured.statusCode).toBe(302);
expect(captured.location).toBe('/');
}
const { res, captured } = makeRes();
await callRoute(
'get',
'/desktop',
makeReq({
path: '/desktop',
originalUrl: '/desktop?shared=%2Falice%2Fx%2Fy&download=a',
}),
res,
);
expect(captured.location).toBe('/?shared=%2Falice%2Fx%2Fy&download=a');
});
it('redirects /desktop/app/:name to the app landing', async () => {
const { res, captured } = makeRes();
await callRoute(
'get',
'/desktop/app/:name',
makeReq({
params: { name: 'editor' },
path: '/desktop/app/editor',
originalUrl: '/desktop/app/editor?file=%2Falice%2Fa.txt',
}),
res,
);
expect(captured.statusCode).toBe(302);
expect(captured.location).toBe('/app/editor?file=%2Falice%2Fa.txt');
expect(html).toContain('launch_app');
expect(html).toContain('explorer');
expect(html).toContain('/alice/Documents');
});
});
@@ -25,6 +25,7 @@ import type { PuterRouter } from '../../core/http/PuterRouter';
import type {
PuterHomepageService,
PageMeta,
LaunchOptions,
} from '../../services/homepage/PuterHomepageService';
/** Longest description/og:description/twitter:description on `/app/<name>`. */
@@ -42,9 +43,9 @@ const APP_LANDING_LIMIT = {
};
/**
* App descriptions can run long; search snippets and social cards want a short
* blurb. Collapses whitespace, then cuts at a word boundary with an ellipsis so
* the result never exceeds `APP_META_DESCRIPTION_MAX`.
* App descriptions can run long; search snippets and social cards want a
* short blurb. Collapses whitespace, then cuts at a word boundary with an
* ellipsis so the result never exceeds `APP_META_DESCRIPTION_MAX`.
*/
export function appMetaDescription(text: string): string {
const clean = text.replace(/\s+/g, ' ').trim();
@@ -88,6 +89,7 @@ export class HomepageController extends PuterController {
req: express.Request,
res: express.Response,
metaOverrides: Partial<PageMeta> = {},
launch: LaunchOptions = {},
) => {
const meta = { ...defaultMeta(req), ...metaOverrides };
const actor =
@@ -96,7 +98,7 @@ export class HomepageController extends PuterController {
actor?: Parameters<typeof homepage.send>[0]['actor'];
}
).actor ?? null;
await homepage.send({ req, res, actor }, meta);
await homepage.send({ req, res, actor }, meta, launch);
};
// -- Root + path-aliased shell routes ------------------------
@@ -109,41 +111,17 @@ export class HomepageController extends PuterController {
router.get('/dashboard', {}, (req, res) => sendShell(req, res));
router.get('/dashboard/', {}, (req, res) => sendShell(req, res));
// `/desktop` used to pick the desktop interface. What follows it is
// the same route at the root, so it redirects there, keeping the query
// (`?shared=`, `?download=`); the browser carries the hash over.
const redirectWithoutDesktop = (
req: express.Request,
res: express.Response,
target: string,
) => {
const url = req.originalUrl ?? req.url ?? '';
const query = url.includes('?') ? url.slice(url.indexOf('?')) : '';
res.redirect(302, `${target}${query}`);
};
router.get('/desktop', {}, (req, res) =>
redirectWithoutDesktop(req, res, '/'),
);
router.get('/desktop/', {}, (req, res) =>
redirectWithoutDesktop(req, res, '/'),
);
router.get('/desktop/app/:name', {}, (req, res) =>
redirectWithoutDesktop(
req,
res,
`/app/${encodeURIComponent(String(req.params.name))}`,
),
);
router.get('/desktop', {}, (req, res) => sendShell(req, res));
router.get('/desktop/', {}, (req, res) => sendShell(req, res));
router.get('/action/*splat', {}, (req, res) => sendShell(req, res));
// `/@<user>/<path>` names something in that user's Public folder; the
// GUI opens it.
router.get('/@:username', {}, (req, res) => sendShell(req, res));
router.get('/@:username/*splat', {}, (req, res) => sendShell(req, res));
// -- /app/:name - app metadata baked into the shell ----------
// `/desktop/app/:name` is the same landing booted on the desktop
// instead of the dashboard; the GUI strips the prefix and treats the
// rest of the path identically, so both render the same shell.
const sendAppShell = async (
req: express.Request,
@@ -198,10 +176,27 @@ export class HomepageController extends PuterController {
{ rateLimit: APP_LANDING_LIMIT },
sendAppShell,
);
router.get(
'/desktop/app/:name',
{ rateLimit: APP_LANDING_LIMIT },
sendAppShell,
);
// -- /show/* - the GUI opens the path that follows ----------
// -- /show/* - launch explorer with the requested file path --
router.get('/show/*splat', {}, (req, res) => sendShell(req, res));
router.get('/show/*splat', {}, (req, res) => {
const filePath = req.path.slice('/show'.length);
const launch: LaunchOptions = {
on_initialized: [
{
$: 'window-call',
fn_name: 'launch_app',
args: [{ name: 'explorer', path: filePath }],
},
],
};
return sendShell(req, res, {}, launch);
});
// -- /sw.js - the service worker, scoped to the whole origin --
@@ -19,9 +19,8 @@
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import type { AddressInfo } from 'node:net';
import path from 'node:path';
import express, { type Request, type Response } from 'express';
import type { Request, Response } from 'express';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { PuterRouter } from '../../core/http/PuterRouter.js';
import type { RouteDescriptor } from '../../core/http/types.js';
@@ -112,7 +111,6 @@ const callGetHandler = async (
let captured: { filename?: string; root?: string } = {};
const req = {} as Request;
const res = {
setHeader: vi.fn(),
sendFile: vi.fn((filename: string, opts: { root: string }) => {
captured = { filename, root: opts.root };
}),
@@ -240,48 +238,6 @@ describe('StaticAssetsController puterjs_root', () => {
});
});
// ── push-sw.js caching ──────────────────────────────────────────────
/** Serves the route's handler over real HTTP and returns the response. */
const fetchThroughExpress = async (router: PuterRouter, routePath: string) => {
const route = findRoute(router, 'get', routePath);
if (!route) throw new Error(`No GET ${routePath} registered`);
const app = express();
app.get(routePath, (req, res, next) => route.handler(req, res, next));
const listener = app.listen(0);
await new Promise((resolve) => listener.once('listening', resolve));
try {
const { port } = listener.address() as AddressInfo;
return await fetch(`http://127.0.0.1:${port}${routePath}`);
} finally {
listener.close();
}
};
describe('StaticAssetsController push-sw.js', () => {
it.each([
['client_libs_root', 'puter.js/push-sw.js'],
['puterjs_root', 'push-sw.js'],
] as const)(
'lets the CDN keep it for five minutes from %s',
async (configKey, file) => {
const root = makeTempDir();
mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
writeFileSync(path.join(root, file), '/* worker */');
const router = buildController({ [configKey]: root });
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
const response = await fetchThroughExpress(router, route);
expect(response.status).toBe(200);
expect(response.headers.get('cache-control')).toBe(
'public, max-age=0, s-maxage=300',
);
expect(await response.text()).toBe('/* worker */');
}
},
);
});
// ── gui_assets_root ─────────────────────────────────────────────────
describe('StaticAssetsController gui_assets_root', () => {
@@ -23,9 +23,6 @@ import path from 'node:path';
import { PuterController } from '../types.js';
import type { PuterRouter } from '../../core/http/PuterRouter';
// Sites import push-sw.js by URL, so the CDN refetches it within five minutes.
const PUSH_WORKER_CACHE_CONTROL = 'public, max-age=0, s-maxage=300';
/**
* Static asset routes.
*
@@ -61,7 +58,6 @@ export class StaticAssetsController extends PuterController {
// `/v2/push-sw.js` is the path older docs gave.
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
router.get(route, { subdomain: 'js' }, (_req, res) => {
res.setHeader('Cache-Control', PUSH_WORKER_CACHE_CONTROL);
res.sendFile('puter.js/push-sw.js', { root });
});
}
@@ -112,10 +108,6 @@ export class StaticAssetsController extends PuterController {
});
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
router.get(route, { subdomain: 'js' }, (_req, res) => {
res.setHeader(
'Cache-Control',
PUSH_WORKER_CACHE_CONTROL,
);
res.sendFile('push-sw.js', { root: puterjsRoot });
});
}
-42
View File
@@ -1,42 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { describe, expect, it } from 'vitest';
import { isHttpError } from './http/HttpError.js';
import {
SCHEDULE_LIST_LIMIT_CAP,
SCHEDULE_PEER_PATH,
schedulesUnavailable,
} from './schedules.js';
describe('schedules contract', () => {
it('answers an absent schedules service with a 503 events_disabled', () => {
const err = schedulesUnavailable();
expect(isHttpError(err)).toBe(true);
expect(err.statusCode).toBe(503);
expect(err.legacyCode).toBe('events_disabled');
// Expected on a server without schedules, so it must not page.
expect(err.noAlarm).toBe(true);
});
it('keeps the list cap and the peer path the extension relies on', () => {
expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
});
});
-190
View File
@@ -1,190 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* What core asks of schedules. Nothing here is an implementation: an extension
* registers `services.schedules`, and a server without one answers every
* schedule route and `notify({ sendAt })` with `events_disabled`. Called by
* extensions; keep these signatures stable.
*/
import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
import type { AppMessage } from '../services/notification/appMessage.js';
import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
import type { DurableSubscription } from '../stores/events/types.js';
import type { PageResult } from '../util/pagination.js';
import type { Actor } from './actor.js';
import { HttpError } from './http/HttpError.js';
/** Why a schedule ended. */
export type ScheduleEndReason =
| 'removed'
| 'ended'
| 'app_deleted'
| 'uninstalled'
| 'permission_revoked'
| 'holder_gone'
| 'app_gone'
| 'orphaned'
| 'delivered'
| 'dropped';
/** What happened to a set of subscription rows schedules own. */
export type ScheduleSubscriptionChange =
| {
type: 'suspended';
reason: SuspendedReason;
/** For a reason that never lifts; defaults to that reason. */
endReason?: ScheduleEndReason;
}
| { type: 'resumed' }
| { type: 'ended'; endReason: ScheduleEndReason };
/** A schedule that keeps running, one that runs once, or a delayed notification. */
export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
/**
* A schedule as its holder sees it. Times are epoch ms. `context` values are
* never returned, as for a persistent subscription: only its keys and a hash.
*/
export interface ScheduleView {
scheduleId: string;
kind: ScheduleViewKind;
appUid: string;
/** `null` once ended: the handler binding goes with the schedule. */
handlerName: string | null;
/** Set only when the handler lives in the holder's own scope. */
handlerScope?: 'user';
contextKeys: string[] | null;
contextHash: string | null;
/** Interval of an `every` schedule. */
every: number | null;
cron: string | null;
tz: string;
/** First allowed run; a one-shot's only run. */
startAt: number;
/** Last allowed run, inclusive. */
endsAt: number | null;
/** `sending` once a delayed send has started; it can't be cancelled then. */
state: 'active' | 'suspended' | 'sending' | 'ended';
suspendedReason: string | null;
endReason: string | null;
endedAt: number | null;
/**
* The slot it runs for next; `null` while suspended or sending, and once
* ended.
*/
nextRunAt: number | null;
createdAt: number;
}
export interface ScheduleListRequest {
limit: number;
cursor?: string;
includeTotal: boolean;
}
/** The caps a holder's schedules are held to, from their plan. */
export interface ScheduleQuota {
recurringPerUser: number;
recurringPerApp: number;
oneShotsPerUser: number;
oneShotsPerApp: number;
/** Worst-case runs per hour × 1000, summed over recurring rows. */
budgetPerUser: number;
budgetPerApp: number;
}
/** A caller cleared to supply `sendAt`, and when it asked for. */
export interface DeferredSendPlan {
sendAt: number;
/** Due within 5 seconds, or past: sent now rather than stored. */
immediate: boolean;
holderUserId: number;
appUid: string;
handlerDepth: number;
quota: ScheduleQuota;
}
/** What a delayed app message stores until it is sent. */
export interface DeferredAppMessage {
message: AppMessage;
/** A KV share handle naming the recipient. */
to?: string;
}
/**
* A condition on one row of another table that statements embed as `EXISTS
* (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
*/
export interface SqlGuard {
table: string;
where: string;
values: unknown[];
}
/** Ceiling of a schedule listing page. */
export const SCHEDULE_LIST_LIMIT_CAP = 200;
/** The signed peer route a region's schedule changes are sent to. */
export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
export const schedulesUnavailable = (): HttpError =>
new HttpError(503, 'Schedules are not available on this server', {
legacyCode: 'events_disabled',
// The designed answer of a server without schedules, not a fault.
noAlarm: true,
});
export interface ScheduleServiceContract {
/**
* Keep schedules in step with a change to the subscription rows they own.
* Never throws: the subscription change has already happened.
*/
onSubscriptionsChanged(
rows: readonly DurableSubscription[],
change: ScheduleSubscriptionChange,
): Promise<void>;
createFor(
actor: Actor,
body: Record<string, unknown>,
): Promise<ScheduleView>;
list(
actor: Actor,
request: ScheduleListRequest,
): Promise<PageResult<ScheduleView>>;
get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
remove(actor: Actor, scheduleId: unknown): Promise<void>;
receivePeerChanges(body: unknown): Promise<{ applied: number }>;
/** Checks a caller supplying `sendAt`; `events_disabled` when off. */
prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
createDeferredAppMessage(
plan: DeferredSendPlan,
message: DeferredAppMessage,
): Promise<{ scheduleId: string; sendAt: number }>;
/**
* Runs an events worker deploy under the schedules' deploy gate; `capped`
* for a schedule run's. A passthrough when schedules are off.
*/
gateDeploy?(
script: string,
options: { capped: boolean },
deploy: () => Promise<EventsDeployOutcome>,
): Promise<EventsDeployOutcome>;
}
+18 -2
View File
@@ -31,7 +31,7 @@ import {
normalizeRawBase64ImageString,
validateIconDataUrl,
} from '../../util/appIcon.js';
import { isOriginBootstrapApp, toAppReadView } from '../../util/appView.js';
import { toAppReadView } from '../../util/appView.js';
import { awaitLater } from '../../util/concurrency.js';
import { isUniqueViolation } from '../../util/dbError.js';
import {
@@ -1088,6 +1088,22 @@ export class AppDriver extends PuterDriver {
}
}
/**
* Origin-bootstrap detection: rows auto-created when an unknown origin
* first needed an app row (no human-supplied metadata). Marker is `name ===
* uid && title === uid` and a description starting with "App created from
* origin ". Only these rows are eligible for same-owner merging — refusing
* to merge arbitrary same-owner apps prevents accidental data loss.
*/
#isOriginBootstrapApp(app) {
if (!app || typeof app !== 'object') return false;
if (typeof app.uid !== 'string' || !app.uid) return false;
if (app.name !== app.uid) return false;
if (app.title !== app.uid) return false;
if (typeof app.description !== 'string') return false;
return app.description.startsWith('App created from origin ');
}
// -- Canonical-uid alias kvstore pair -----------------------------
//
// After a merge, the source app uid → canonical uid mapping is
@@ -1250,7 +1266,7 @@ export class AppDriver extends PuterDriver {
if (
Number.isInteger(conflictOwnerUserId) &&
conflictOwnerUserId === user.id &&
!isOriginBootstrapApp(appToJoin)
!this.#isOriginBootstrapApp(appToJoin)
) {
// Prevent merging arbitrary same-owner apps; only allow the
// auto-created origin bootstrap row to be absorbed.
@@ -1,493 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* `notify({ sendAt })` without and with `services.schedules`, and the deferred
* mailbox write an extension runs inside its own transaction, fenced on a guard
* in a table of its own (a test table here).
*/
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
import { makeActor, type Actor } from '../../core/actor.js';
import { runWithContext } from '../../core/context.js';
import { isHttpError } from '../../core/http/HttpError.js';
import type {
DeferredSendPlan,
ScheduleServiceContract,
} from '../../core/schedules.js';
import type { PuterServer } from '../../server.js';
import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
import { setupTestServer } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import type {
DeferredWriteTx,
NotificationDriver,
} from './NotificationDriver.js';
vi.setConfig({ testTimeout: 120_000 });
const BOOT_TIMEOUT_MS = 480_000;
const TEST_TABLE = 'deferred_send_test';
let server: PuterServer;
let holder: Actor;
let app: { id: number; uid: string };
let saved: unknown;
let nextId = 1;
const db = () => server.clients.db;
const driver = () =>
server.drivers.notifications as unknown as NotificationDriver;
const slot = () => server.services as unknown as { schedules?: unknown };
/** Put `schedules` in the slot, or leave the slot empty. */
const setSchedules = (schedules: unknown): void => {
if (schedules === undefined) delete slot().schedules;
else slot().schedules = schedules;
};
const asApp = (): Actor => makeActor({ user: holder.user, app });
const notify = (fields: Record<string, unknown>): Promise<unknown> =>
runWithContext({ actor: asApp() }, () =>
driver().create({
object: { value: { type: 'app.message', title: 'Hi', ...fields } },
}),
);
const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
try {
await run;
} catch (err) {
if (isHttpError(err)) return err.legacyCode;
throw err;
}
return undefined;
};
const mailbox = () =>
db().read(
'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
'ORDER BY `id`',
[holder.user!.id, app.uid],
);
/** A row's value: JSON text on sqlite, parsed on the other engines. */
const valueOf = (row: Record<string, unknown>) =>
typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
/** A send claimed under `token`, as an extension's own row would be. */
const claimed = async (token: string): Promise<number> => {
const id = nextId++;
await db().write(
`INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
[id, token],
);
return id;
};
/** The transaction an extension wraps the write in: its row, then the mailbox. */
const txFor = (id: number, token: string): DeferredWriteTx => ({
guard: {
table: TEST_TABLE,
where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
values: [id, token],
},
async commit(entries: readonly BatchEntry[]) {
await db().batchWrite([
{
statement:
`UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
"WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
values: [id, token],
},
...entries,
]);
const [row] = await db().pread(
`SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
[id],
);
return row?.state === 'delivered' && row.token === token;
},
});
const write = (
sendKey: string,
tx: DeferredWriteTx,
over: {
payload?: string | null;
appUid?: string;
signal?: AbortSignal;
} = {},
) =>
driver().deferredAppMessages.write(
{
holderUserId: holder.user!.id,
appUid: over.appUid ?? app.uid,
payload:
over.payload === undefined
? JSON.stringify({ message: { title: 'Later' } })
: over.payload,
sendKey,
signal: over.signal ?? new AbortController().signal,
},
tx,
);
const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
({
onSubscriptionsChanged: vi.fn(async () => {}),
createFor: vi.fn(),
list: vi.fn(),
get: vi.fn(),
remove: vi.fn(),
receivePeerChanges: vi.fn(),
prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
sendAt: 0,
immediate: false,
holderUserId: holder.user!.id,
appUid: app.uid,
handlerDepth: 0,
quota: {
recurringPerUser: 1,
recurringPerApp: 1,
oneShotsPerUser: 1,
oneShotsPerApp: 1,
budgetPerUser: 1,
budgetPerApp: 1,
},
...plan,
})),
createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
scheduleId: 'sched-1',
sendAt: p.sendAt,
})),
}) satisfies ScheduleServiceContract;
beforeAll(async () => {
server = await setupTestServer({
events: { enabled: true },
} as IConfig);
const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
const user = await server.stores.user.create({
username,
uuid: uuidv4(),
password: null,
email: `${username}@test.local`,
requires_email_confirmation: false,
});
const row = (await server.stores.user.getById(user.id))!;
holder = makeActor({
user: {
id: row.id,
uuid: row.uuid,
username: row.username,
email: row.email ?? null,
email_confirmed: true,
} as Actor['user'],
});
const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
app = (await server.stores.app.create(
{ name, title: name, index_url: `https://${name}.test/` },
{ ownerUserId: row.id },
)) as { id: number; uid: string };
await server.services.permission.grantUserAppPermission(
holder,
app.uid,
EVENTS_BACKGROUND_PERMISSION,
);
await db().write(
`CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
'`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
[],
);
saved = slot().schedules;
}, BOOT_TIMEOUT_MS);
beforeEach(async () => {
await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
app.uid,
]);
});
afterEach(() => {
setSchedules(saved);
vi.restoreAllMocks();
});
afterAll(async () => {
await server?.shutdown();
});
describe('notify({ sendAt }) without services.schedules', () => {
it('is refused events_disabled, even when due now, storing nothing', async () => {
setSchedules(undefined);
expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
'events_disabled',
);
expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
'events_disabled',
);
expect(await mailbox()).toEqual([]);
});
it('sends as before without sendAt', async () => {
setSchedules(undefined);
const sent = (await notify({ text: 'now' })) as { uid: string };
expect(sent.uid).toBeTruthy();
expect(await mailbox()).toHaveLength(1);
});
});
describe('notify({ sendAt }) with services.schedules', () => {
it('hands a later send to createDeferredAppMessage and answers its id', async () => {
const at = Date.now() + 3_600_000;
const schedules = fakeSchedules({ sendAt: at });
setSchedules(schedules);
await expect(notify({ sendAt: at, text: 'later' })).resolves.toEqual({
scheduled: { scheduleId: 'sched-1', sendAt: at },
});
expect(schedules.prepareSendAt).toHaveBeenCalledWith(
expect.objectContaining({ user: holder.user }),
at,
);
expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
expect.objectContaining({ sendAt: at }),
{
message: expect.objectContaining({
title: 'Hi',
text: 'later',
}),
},
);
expect(await mailbox()).toEqual([]);
});
it('sends at once when the plan says it is due', async () => {
const schedules = fakeSchedules({ immediate: true });
setSchedules(schedules);
const sent = (await notify({ sendAt: Date.now() })) as {
uid: string;
};
expect(sent.uid).toBeTruthy();
expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
expect(await mailbox()).toHaveLength(1);
});
it('passes a refusal from prepareSendAt through', async () => {
const schedules = fakeSchedules({});
schedules.prepareSendAt.mockRejectedValueOnce(
Object.assign(new Error('scoped'), {}),
);
setSchedules(schedules);
await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
expect(await mailbox()).toEqual([]);
});
});
describe('deferredAppMessages.write', () => {
it('writes one row keyed by the send, in the caller`s transaction', async () => {
const token = uuidv4();
const key = uuidv4();
const id = await claimed(token);
await expect(write(key, txFor(id, token))).resolves.toBe('delivered');
const rows = await mailbox();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ uid: key, send_key: key });
expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
});
it('writes nothing when the guard no longer holds: another claim took it', async () => {
const id = await claimed(uuidv4());
const key = uuidv4();
await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
expect(await mailbox()).toEqual([]);
});
it('replaces the open row of its group, which takes the key', async () => {
const open = (await notify({ group: 'g', text: 'first' })) as {
uid: string;
};
const token = uuidv4();
const key = uuidv4();
const id = await claimed(token);
await expect(
write(key, txFor(id, token), {
payload: JSON.stringify({
message: { title: 'Later', group: 'g' },
}),
}),
).resolves.toBe('delivered');
const rows = await mailbox();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
});
it('drops a payload it can`t read, and a sender whose app is gone', async () => {
const token = uuidv4();
const id = await claimed(token);
await expect(
write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
).resolves.toBe('dropped');
await expect(
write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
).resolves.toBe('dropped');
expect(await mailbox()).toEqual([]);
});
it('writes nothing once the caller`s claim is lost', async () => {
const token = uuidv4();
const id = await claimed(token);
const lost = new AbortController();
lost.abort();
await expect(
write(uuidv4(), txFor(id, token), { signal: lost.signal }),
).rejects.toThrow();
expect(await mailbox()).toEqual([]);
});
});
/**
* A second claim taking a send over while the first one's transaction is open.
* Needs two writers at once with row locks; sqlite serialises writers, so this
* runs on postgres only.
*/
describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
'a deferred write racing another writer',
() => {
/**
* Hold the first statement matching `match` on its connection, before
* or after it runs, until released.
*/
const holdStatement = (match: RegExp, when: 'before' | 'after') => {
const client = db() as unknown as {
query: (
target: unknown,
sql: string,
params?: unknown[],
) => Promise<unknown>;
};
const original = client.query.bind(client);
let reached!: () => void;
let release!: () => void;
const atBarrier = new Promise<void>((r) => (reached = r));
const released = new Promise<void>((r) => (release = r));
let held = false;
vi.spyOn(client as never, 'query' as never).mockImplementation(
(async (target: unknown, sql: string, params?: unknown[]) => {
if (held || !match.test(sql))
return original(target, sql, params);
held = true;
if (when === 'before') {
reached();
await released;
return original(target, sql, params);
}
const result = await original(target, sql, params);
reached();
await released;
return result;
}) as never,
);
return { atBarrier, release };
};
const DELIVERED = /SET `state` = 'delivered'/;
const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
/** Whether `promise` is still pending after `ms`. */
const stillPending = async (promise: Promise<unknown>, ms: number) => {
let settled = false;
const watched = (async () => {
try {
await promise;
} finally {
settled = true;
}
})();
await new Promise((r) => setTimeout(r, ms));
void watched;
return !settled;
};
it('a transaction that opened before a takeover writes nothing', async () => {
const stale = uuidv4();
const id = await claimed(stale);
const key = uuidv4();
const gate = holdStatement(DELIVERED, 'before');
const first = write(key, txFor(id, stale));
await gate.atBarrier;
await db().write(
`UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
[uuidv4(), id],
);
gate.release();
await expect(first).resolves.toBe('lost');
expect(
await server.stores.notification.getBySendKey(key),
).toBeNull();
expect(await mailbox()).toEqual([]);
});
it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
const open = (await notify({ group: 'g', text: 'first' })) as {
uid: string;
};
const token = uuidv4();
const id = await claimed(token);
const key = uuidv4();
const gate = holdStatement(REPLACE, 'after');
const deferred = write(key, txFor(id, token), {
payload: JSON.stringify({
message: { title: 'Later', group: 'g', text: 'later' },
}),
});
await gate.atBarrier;
const immediate = notify({ group: 'g', text: 'newer' });
// Past the group lock's wait, then blocked on the row.
expect(await stillPending(immediate, 2_500)).toBe(true);
gate.release();
await expect(deferred).resolves.toBe('delivered');
await immediate;
const rows = await mailbox();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
expect(valueOf(rows[0]).text).toBe('newer');
});
},
);
@@ -51,14 +51,6 @@ import {
resolveNotificationWrite,
type NotificationTypeName,
} from '../../services/notification/notificationTypes.js';
import {
schedulesUnavailable,
type DeferredAppMessage,
type DeferredSendPlan,
type ScheduleServiceContract,
type SqlGuard,
} from '../../core/schedules.js';
import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
import { withRedisLock } from '../../util/redisLock.js';
import { PuterDriver } from '../types.js';
import type { Actor } from '../../core/actor.js';
@@ -75,44 +67,6 @@ const GROUP_LOCK_RETRY_MS = 25;
type AppMessageWindow = 'minute' | 'day';
export type { SqlGuard };
/** What a delayed app message's fire-time write needs from its row. */
export interface DeferredWriteInput {
holderUserId: number;
appUid: string;
/** The stored {@link DeferredAppMessage}, as JSON text. */
payload: string | null;
/** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
sendKey: string;
/** Aborted when the caller's claim is lost. */
signal: AbortSignal;
}
/**
* The caller's transaction the mailbox write commits in. Atomicity is the
* caller's to keep; the type can't enforce it.
*/
export interface DeferredWriteTx {
/**
* What every mailbox statement is fenced on: the caller's row marked
* delivered by this claim.
*/
guard: SqlGuard;
/**
* Must be the whole delivery in one step: the caller's guarded transition
* of its row to delivered and `entries`, in that order, in one
* `batchWrite`, then a read of the row on the primary. True only when that
* read shows this claim holding the row as delivered. Splitting the
* transition from `entries` lets one slot write two rows.
*/
commit(entries: readonly BatchEntry[]): Promise<boolean>;
}
/** How a delayed send's write stood when it stopped. */
export type DeferredWriteOutcome =
'delivered' | 'dropped' | 'lost' | 'senderSuspended';
/** The mailbox slice a call named, in the terms the query takes it. */
interface MailboxScope {
audiences: readonly string[];
@@ -170,21 +124,6 @@ const parseNotifyTarget = (to: unknown): string => {
return target.handle;
};
/** A stored delayed message, or `null` for one that can't be read back. */
const parseDelayedAppMessage = (
payload: string | null,
): DeferredAppMessage | null => {
try {
const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
if (typeof stored?.message?.title !== 'string') return null;
if (stored.to !== undefined && typeof stored.to !== 'string')
return null;
return stored;
} catch {
return null;
}
};
/** Whether a row belongs to the slice, as the same SQL scope selected it. */
const inScope = (
row: Record<string, unknown>,
@@ -415,7 +354,7 @@ export class NotificationDriver extends PuterDriver {
object: Record<string, unknown>,
value: Record<string, unknown>,
): Promise<unknown> {
const { type, to, sendAt, ...fields } = value;
const { type, to, ...fields } = value;
if (type !== APP_MESSAGE_TYPE) {
throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
}
@@ -431,12 +370,6 @@ export class NotificationDriver extends PuterDriver {
const handle = to === undefined ? undefined : parseNotifyTarget(to);
const message = parseAppMessage(fields);
assertValueFits(value);
const timing =
sendAt === undefined || sendAt === null
? null
: await this.#schedules().prepareSendAt(actor, sendAt);
if (timing && !timing.immediate)
return this.#scheduleAppMessage(actor, message, handle, timing);
if (handle === undefined) {
const uid = await this.#sendAppMessage(
@@ -477,23 +410,12 @@ export class NotificationDriver extends PuterDriver {
appUid: string,
message: AppMessage,
): Promise<string> {
return this.#oneSendPerGroup(recipientId, appUid, message, () =>
this.#replaceOrWriteAppMessage(recipientId, appUid, message),
);
}
/**
* One send per group at a time, so a burst into an empty group writes one
* row that the rest replace. A lock that is busy past the wait or
* unreachable still sends: the race costs a second row and its day slot.
*/
#oneSendPerGroup<T>(
recipientId: number,
appUid: string,
message: AppMessage,
send: () => Promise<T>,
): Promise<T> {
const send = () =>
this.#replaceOrWriteAppMessage(recipientId, appUid, message);
if (!message.group) return send();
// One send per group at a time, so a burst into an empty group writes
// one row that the rest replace. A lock that is busy past the wait or
// unreachable still sends: the race costs a second row and its day slot.
return withRedisLock(
this.clients.redis,
`notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
@@ -508,163 +430,6 @@ export class NotificationDriver extends PuterDriver {
);
}
// -- Delayed sends -----------------------------------------------
#schedules(): ScheduleServiceContract {
const schedules = this.services.schedules;
if (!schedules) throw schedulesUnavailable();
return schedules;
}
/**
* Store an app's message to go out at `sendAt`. A recipient named through a
* share is checked now without spending its budget; every limit is spent
* when it goes, and the share is checked again then.
*/
async #scheduleAppMessage(
actor: UserActor,
message: AppMessage,
handle: string | undefined,
plan: DeferredSendPlan,
): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
if (handle !== undefined)
await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
spendBudget: false,
});
return {
scheduled: await this.#schedules().createDeferredAppMessage(
plan,
handle === undefined ? { message } : { message, to: handle },
),
};
}
/**
* For extensions. A field, not a method, so `/drivers/call` can't reach it.
*/
readonly deferredAppMessages = {
write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
this.#writeDeferredAppMessage(input, tx),
};
/**
* Check the sender and recipient as they stand now, spend the recipient's
* limits, and write the mailbox row through `tx`, fenced on its guard.
* Refusals are the same `HttpError`s an immediate send throws.
*/
async #writeDeferredAppMessage(
input: DeferredWriteInput,
tx: DeferredWriteTx,
): Promise<DeferredWriteOutcome> {
const { holderUserId, sendKey, signal } = input;
const sender = await this.services.events.resolveScheduleActor(
holderUserId,
input.appUid,
);
if (!sender) {
// A lagging replica can miss an app the primary has: retried.
const there =
(await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
(await this.stores.user.getById(holderUserId)) !== null;
if (there) throw new Error('the sender could not be resolved');
return 'dropped';
}
if (sender.user?.suspended) return 'senderSuspended';
// Uninstalled since, and the cancel at uninstall missed it.
const appId = sender.effectiveApp?.id;
if (
typeof appId !== 'number' ||
!(await this.stores.permission.hasAnyUserAppPerm(
holderUserId,
appId,
))
)
return 'dropped';
const stored = parseDelayedAppMessage(input.payload);
if (!stored) return 'dropped';
const { message } = stored;
let recipientId = holderUserId;
let appUid = input.appUid;
if (stored.to !== undefined) {
const recipient =
await this.services.kvShare.resolveNotifyRecipient(
sender,
stored.to,
);
recipientId = recipient.recipientUserId;
appUid = recipient.appUid;
}
return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
// Primary, as for an immediate send.
const open = message.group
? await this.stores.notification.getUnseenByGroup(
recipientId,
appUid,
message.group,
{ primary: true },
)
: null;
await this.#spendAppMessageLimits(
recipientId,
appUid,
open ? ['minute'] : ['minute', 'day'],
);
message.app = appMessageSender(
appUid,
await this.stores.app.getByUid(appUid),
);
// Another attempt may hold the lease now; hand the claim back.
if (signal.aborted) throw new Error('delayed send lost its lease');
const registered = resolveNotificationWrite(
APP_MESSAGE_TYPE,
appUid,
);
const delivered = await tx.commit(
this.stores.notification.sendEntries(
{
sendKey,
userId: recipientId,
value: { ...message, type: registered.type },
type: registered.type,
audience: registered.audience,
appUid,
group: message.group ?? null,
replaceUid: open ? String(open.uid) : null,
at: Date.now(),
},
tx.guard,
),
);
if (!delivered) return 'lost';
if (open)
await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
return 'delivered';
});
}
/**
* A replacement that found its row shown since the lookup wrote a new one,
* which costs a day slot. Already written, so a full day is not a refusal.
*/
async #spendDayIfFresh(
recipientId: number,
appUid: string,
open: Record<string, unknown>,
sendKey: string,
): Promise<void> {
try {
const written =
await this.stores.notification.getBySendKey(sendKey);
if (written && written.uid !== open.uid)
await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
} catch {
// The row is written; a slot not spent is the only loss.
}
}
async #replaceOrWriteAppMessage(
recipientId: number,
appUid: string,
@@ -129,27 +129,6 @@ describe('pipeStreamResult', () => {
expect(warn).not.toHaveBeenCalled();
});
it('adds at most two close listeners to the response', async () => {
// The response already carries close listeners from metering,
// compression and tracing; Node warns past ten.
let added = -1;
server = createServer((_req, res) => {
const before = res.listenerCount('close');
pipeStreamResult(Readable.from(['a', 'b']), res, 'test.run');
added = res.listenerCount('close') - before;
});
await new Promise<void>((resolve) => server!.listen(0, resolve));
const { port } = server.address() as AddressInfo;
await new Promise<void>((resolve) =>
get(`http://127.0.0.1:${port}/`, (res) => {
res.resume();
res.on('end', () => resolve());
}),
);
expect(added).toBeGreaterThan(0);
expect(added).toBeLessThanOrEqual(2);
});
it('drops the source, without throwing, when the client left before the result was ready', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const source = Readable.from(['late']);
+8 -16
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type { Readable } from 'node:stream';
import { pipeline, type Readable } from 'node:stream';
// Only what's used here, so callers on another @types/node (or express's
// Response) still fit.
@@ -32,32 +32,24 @@ type ResponseSink = NodeJS.WritableStream & {
* mid-body destroys the response, so the client sees an aborted transfer rather
* than a clean end, and the failure is logged once instead of surfacing as an
* unhandled stream error. A client that hangs up stops the source.
*
* `pipe` rather than `pipeline`: pipeline puts seven close listeners on the
* response, and with the ones metering, compression and tracing add, every
* streamed response tripped Node's ten-listener warning.
*/
export function pipeStreamResult(
stream: Readable,
res: ResponseSink,
label: string,
): void {
// The caller hung up while the driver ran: nothing will read the source.
// The caller hung up while the driver ran; `pipeline` throws on a closed
// destination, so drop the source instead.
if (res.closed || res.destroyed) {
stream.destroy();
return;
}
stream.once('error', (err) => {
stream.unpipe(res);
pipeline(stream, res, (err) => {
if (!err) return;
if (!res.destroyed) res.destroy();
// The client hanging up is not a failure of ours.
if ((err as { code?: string }).code === 'ERR_STREAM_PREMATURE_CLOSE')
return;
console.warn(`[drivers] ${label} stream failed mid-response:`, err);
});
// A response that closes before the source ended is a client hanging up.
res.once('close', () => {
if (stream.readableEnded) return;
stream.unpipe(res);
stream.destroy();
});
res.once('error', () => stream.destroy());
stream.pipe(res);
}
@@ -754,50 +754,6 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
expect(putCalls()).toHaveLength(1);
});
const uploadedMetadata = () => {
const [, init] = putCalls().at(-1)!;
const form = (init as RequestInit).body as FormData;
return JSON.parse(form.get('metadata') as string);
};
it('sends limits.cpu_ms in the upload metadata when cpuMs is set', async () => {
const { user, actor } = await makeUser();
const path = `/${user.username}/cpu.worker.js`;
await writeSource(actor, user.id, path, 'export default {}');
const scriptName = `dt-cpu-${user.username}`;
await inCtx(actor, () =>
target.create({
appId: '',
workerName: scriptName,
filePath: path,
[INTERNAL_DEPLOY_TARGET]: { scriptName, cpuMs: 50 },
}),
);
expect(uploadedMetadata().limits).toEqual({ cpu_ms: 50 });
});
it('sends no limits key without cpuMs, and none on an ordinary create', async () => {
const { user, actor } = await makeUser();
const path = `/${user.username}/nocpu.worker.js`;
await writeSource(actor, user.id, path, 'export default {}');
const scriptName = `dt-nocpu-${user.username}`;
await deployTo(actor, path, scriptName);
expect(uploadedMetadata()).not.toHaveProperty('limits');
const name = `plain-${user.username}`;
await inCtx(actor, () =>
target.create({ appId: '', workerName: name, filePath: path }),
);
const metadata = uploadedMetadata();
expect(metadata).not.toHaveProperty('limits');
expect(Object.keys(metadata)).toEqual([
'body_part',
'compatibility_flags',
'compatibility_date',
'bindings',
]);
});
it('refuses a file another user owns, even with manage on it', async () => {
const owner = await makeUser();
const deployer = await makeUser();
@@ -3,7 +3,6 @@
// however the interactions with the Puter API are
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
@@ -14,7 +13,6 @@ import {
import { Actor, makeActor as resolveActor } from '../../core/actor.js';
import { runWithContext } from '../../core/context.js';
import { PuterServer } from '../../server.js';
import { DEFAULT_FREE_SUBSCRIPTION } from '../../services/metering/consts.js';
import {
setupPuterTestEnv,
setupTestServer,
@@ -799,174 +797,6 @@ describe('WorkerDriver', () => {
});
});
describe('rented-unit cap', () => {
const FREE_CAP = 16;
const PAID_CAP = 100;
// The cap sits after the deploy-credentials gate, so the driver needs
// credentials to reach it; a create under the cap then fails on the
// missing source file, which is what tells it past the cap check.
let originalWorkersConfig: unknown;
let originalEvents: unknown;
beforeAll(() => {
const driverConfig = (
target as unknown as { config: Record<string, unknown> }
).config;
originalEvents = driverConfig.events;
driverConfig.events = { userHandlers: true };
originalWorkersConfig = driverConfig.workers;
driverConfig.workers = {
XAUTHKEY: 'test-key',
ACCOUNTID: 'test-account',
};
});
afterAll(() => {
const driverConfig = (
target as unknown as { config: Record<string, unknown> }
).config;
driverConfig.workers = originalWorkersConfig;
driverConfig.events = originalEvents;
});
const holding = (
plan: string,
held: { workers?: number; scopes?: number; owned?: number },
) => {
vi.spyOn(
server.services.metering,
'getActorSubscription',
).mockResolvedValue({ id: plan } as never);
vi.spyOn(
server.stores.subdomain,
'countByUserIdAndPrefix',
).mockResolvedValue(held.workers ?? 0);
vi.spyOn(
server.stores.eventHandler,
'countUserScopesWithHandlersForUser',
).mockResolvedValue(held.scopes ?? 0);
vi.spyOn(
server.stores.eventHandler,
'countAppsWithHandlersForOwner',
).mockResolvedValue(held.owned ?? 0);
};
afterEach(() => {
vi.restoreAllMocks();
});
const createWorker = () =>
inCtx(() =>
target.create({
appId: 'test-app',
workerName: 'capped',
filePath: '/test.js',
}),
);
it('keeps the flat 100-worker cap over ordinary workers with user scopes off', async () => {
const driverConfig = (
target as unknown as { config: Record<string, unknown> }
).config;
driverConfig.events = { userHandlers: false };
try {
// 16 workers is fine here, and scopes and owned apps don't count.
holding(DEFAULT_FREE_SUBSCRIPTION, {
workers: 16,
scopes: 50,
owned: 50,
});
const list = vi
.spyOn(server.stores.subdomain, 'listByUserIdAndPrefix')
.mockResolvedValue(
Array.from({ length: 16 }, () => ({})) as never,
);
await expect(createWorker()).rejects.not.toMatchObject({
statusCode: 403,
});
list.mockResolvedValue(
Array.from({ length: 100 }, () => ({})) as never,
);
await expect(createWorker()).rejects.toMatchObject({
statusCode: 403,
message: 'Worker limit reached (max 100)',
legacyCode: 'forbidden',
});
} finally {
driverConfig.events = { userHandlers: true };
}
});
it('refuses a free user one unit past the cap, across all three sources', async () => {
for (const held of [
{ workers: FREE_CAP },
{ scopes: FREE_CAP },
{ owned: FREE_CAP },
{ workers: 10, scopes: 4, owned: 2 },
]) {
holding(DEFAULT_FREE_SUBSCRIPTION, held);
await expect(createWorker()).rejects.toMatchObject({
statusCode: 403,
legacyCode: 'forbidden',
});
}
});
it('lets a free user create up to the cap', async () => {
holding(DEFAULT_FREE_SUBSCRIPTION, {
workers: 10,
scopes: 3,
owned: 2,
});
await expect(createWorker()).rejects.not.toMatchObject({
statusCode: 403,
});
});
it('refuses a paid user the unit after 100', async () => {
holding('paid-plan', { workers: 90, scopes: 5, owned: 5 });
await expect(createWorker()).rejects.toMatchObject({
statusCode: 403,
});
holding('paid-plan', { workers: 90, scopes: 5, owned: 4 });
await expect(createWorker()).rejects.not.toMatchObject({
statusCode: 403,
});
expect(PAID_CAP).toBe(100);
});
it('keeps a free user over the cap, redeploys what they hold, and refuses a new one', async () => {
holding(DEFAULT_FREE_SUBSCRIPTION, { workers: 20 });
const user = actor.user;
const exists = vi
.spyOn(server.stores.subdomain, 'getBySubdomain')
.mockImplementation(
async (name: string) =>
(name === 'workers.puter.capped'
? { user_id: user.id, uuid: 'existing-uuid' }
: null) as never,
);
const noApp = makeActor({ app: undefined });
const create = (workerName: string) =>
inCtx(
() =>
target.create({
appId: '',
workerName,
filePath: '/test.js',
}),
noApp,
);
await expect(create('capped')).rejects.not.toMatchObject({
statusCode: 403,
});
await expect(create('another')).rejects.toMatchObject({
statusCode: 403,
});
exists.mockRestore();
});
});
describe('admission bypass', () => {
const unverified = (): Actor =>
makeActor({
+22 -38
View File
@@ -43,7 +43,6 @@ import { PuterDriver } from '../types.js';
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
import { awaitLater } from '../../util/concurrency.js';
import { isUniqueViolation } from '../../util/dbError.js';
import { assertRentedUnitRoom } from '../../util/rentedUnits.js';
import { loadFileInput } from '../util/fileInput.js';
import {
normalizeLimit,
@@ -58,7 +57,6 @@ const CF_BASE_URL = 'https://api.cloudflare.com/client/v4/accounts';
export const WORKER_COMPATIBILITY_DATE = '2025-07-15';
export const WORKER_COMPATIBILITY_FLAGS = ['global_fetch_strictly_public'];
const WORKER_NAME_REGEX = /^[a-zA-Z0-9_-]+$/;
/** Cap on ordinary workers while user-scope handlers are off. */
const MAX_WORKERS_PER_USER = 100;
/**
@@ -181,17 +179,12 @@ export interface InternalDeployTarget {
* scripts belong to what.
*/
tags?: string[];
/**
* Per-invocation CPU limit in milliseconds, sent with the upload. Unset
* sends no limit. Ignored by the local runtime.
*/
cpuMs?: number;
}
const MAX_SOURCE_SIZE = 10 * 1024 * 1024; // 10 MB
// How far to scan an app's child apps when resolving which workers it may
// see. A user is capped at their plan's rented units, so only that many
// child apps can actually own a worker; the ceiling is generous slack over that.
// see. A user is capped at MAX_WORKERS_PER_USER workers, so only that many
// child apps can actually own one; the ceiling is generous slack over that.
const CHILD_APP_SCAN_LIMIT = 1000;
// -- Preamble --------------------------------------------------------
@@ -459,34 +452,26 @@ export class WorkerDriver extends PuterDriver {
this.#requireCfConfig();
// With user scopes on, a new worker is one more rented unit under the
// plan's cap, and redeploying an existing worker is never refused.
// Deploy targets create no worker row and are bounded by their caller.
if (this.config.events?.userHandlers === true) {
if (!skipAdmission && !deployTarget && !existingSub) {
await assertRentedUnitRoom(
{
stores: this.stores,
metering: this.services.metering,
},
actor,
);
}
} else {
const existingWorkers =
skipAdmission || deployTarget
? []
: await this.stores.subdomain.listByUserIdAndPrefix(
actor.user.id,
WORKER_SUBDOMAIN_PREFIX,
);
if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
throw new HttpError(
403,
`Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
{ legacyCode: 'forbidden' },
);
}
// Quota check — count existing workers.puter.* subdomains owned by user
//
// A deploy target creates no such row, so this count can neither see
// its scripts nor be grown by them; running it would only make an
// unrelated worker quota decide whether these deploy. Whatever bounds
// them has to be counted where they actually live, and belongs to the
// caller that owns their lifecycle.
const existingWorkers =
skipAdmission || deployTarget
? []
: await this.stores.subdomain.listByUserIdAndPrefix(
actor.user.id,
WORKER_SUBDOMAIN_PREFIX,
);
if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
throw new HttpError(
403,
`Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
{ legacyCode: 'forbidden' },
);
}
// Read source file. loadFileInput runs the read-ACL check internally
@@ -955,7 +940,6 @@ export class WorkerDriver extends PuterDriver {
compatibility_flags: WORKER_COMPATIBILITY_FLAGS,
compatibility_date: WORKER_COMPATIBILITY_DATE,
...(target?.tags ? { tags: target.tags } : {}),
...(target?.cpuMs ? { limits: { cpu_ms: target.cpuMs } } : {}),
bindings: [
// A script deployed without a token gets no binding at all,
// rather than an empty one for its code to find.
+53 -67
View File
@@ -36,10 +36,6 @@ import { Controller, Get } from './core/http/decorators.ts';
import { HttpError } from './core/http/HttpError.ts';
import { extension, extensionStore } from './extensions.ts';
import { PuterServer } from './server.ts';
import {
JobScheduler,
type JobDefinition,
} from './services/jobs/JobScheduler.ts';
import {
allocateEphemeralPort,
createTestUser,
@@ -476,6 +472,59 @@ describe('PuterServer host header validation — permissive modes', () => {
});
});
describe('PuterServer behind a trusted proxy', () => {
let server: PuterServer;
let port: number;
beforeAll(async () => {
port = await allocateEphemeralPort();
server = await setupTestServer(
{
port,
domain: 'puter.localhost',
origin: `http://puter.localhost:${port}`,
api_base_url: `http://api.puter.localhost:${port}`,
allow_all_host_values: false,
allow_no_host_header: false,
custom_domains_enabled: false,
static_hosting_domain: 'site.puter.localhost',
trust_proxy: 1,
} as unknown as IConfig,
{ listen: true },
);
});
afterAll(async () => {
await server?.shutdown();
});
it('serves the main domain whatever X-Forwarded-Host names', async () => {
const res = await rawRequest(port, '/healthcheck', {
host: `puter.localhost:${port}`,
'x-forwarded-host': 'nosuchsite.site.puter.localhost',
});
expect(res.status).toBe(200);
expect(res.headers['x-frame-options']).toBe('SAMEORIGIN');
});
it('keeps a hosted-site host on site hosting', async () => {
const res = await rawRequest(port, '/healthcheck', {
host: `nosuchsite.site.puter.localhost:${port}`,
'x-forwarded-host': 'api.puter.localhost',
});
expect(res.status).toBe(404);
});
it('reads subdomains from Host', async () => {
const res = await rawRequest(port, '/healthcheck', {
host: `api.puter.localhost:${port}`,
'x-forwarded-host': 'puter.localhost',
});
expect(res.status).toBe(200);
expect(res.headers['x-frame-options']).toBeUndefined();
});
});
describe('PuterServer host header validation — custom domains with every host accepted', () => {
let server: PuterServer;
let port: number;
@@ -1543,69 +1592,6 @@ describe('PuterServer lifecycle hooks', () => {
});
});
describe('PuterServer shutdown during prepare', () => {
it('waits for prepare hooks the drain already started before tearing down', async () => {
const port = await allocateEphemeralPort();
const server = await setupTestServer({ port } as unknown as IConfig, {
listen: true,
});
const order: string[] = [];
(server.services as unknown as Record<string, unknown>).slowPrepare = {
onServerPrepareShutdown: async () => {
order.push('prepare:start');
await new Promise((resolve) => setTimeout(resolve, 200));
order.push('prepare:done');
},
onServerShutdown: () => {
order.push('shutdown');
},
};
await server.prepareShutdown({ drainMs: 0 });
// The listener's close callback starts the prepare hooks.
await new Promise((resolve) => setTimeout(resolve, 50));
await server.shutdown();
expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
});
});
/**
* An extension may take over `services.jobs` (a fleet-wide scheduler). Core
* services register their jobs on it at start, so it must keep the built-in's
* first place in the layer.
*/
describe('PuterServer jobs replacement', () => {
it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
const registered: string[] = [];
class ReplacementJobs extends JobScheduler {
override register(name: string, definition: JobDefinition) {
registered.push(name);
super.register(name, definition);
}
}
const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
extension.registerService('jobs', ReplacementJobs as never);
let server: PuterServer | undefined;
try {
server = await setupTestServer();
const { jobs } = server.services;
expect(jobs).toBeInstanceOf(ReplacementJobs);
expect(Object.keys(server.services)[0]).toBe('jobs');
expect(extension.import('service').jobs).toBeInstanceOf(
ReplacementJobs,
);
// Only the replacement gets lifecycle hooks.
expect(started.mock.contexts).toEqual([jobs]);
expect(registered).toContain('auth.sessions.purge');
} finally {
delete (extensionStore.services as Record<string, unknown>).jobs;
started.mockRestore();
await server?.shutdown();
}
});
});
/**
* Each layer's `onServerShutdown` has to run while the layers beneath it (the
* ones it writes through) are still up, so teardown goes top-down: drivers,
+17 -9
View File
@@ -413,6 +413,17 @@ export class PuterServer {
* `#materializeRoute` as those options ship.
*/
#installGlobalMiddleware() {
// -- Forwarded host ------------------------------------------
// With `trust proxy` set, express derives `req.hostname` and
// `req.subdomains` from X-Forwarded-Host, which any client can send.
// Dropping it keeps them on Host, the header host validation checks,
// so a request can't pick a different site or subdomain router than
// the host it was sent to.
this.#app.use((req, _res, next) => {
delete req.headers['x-forwarded-host'];
next();
});
// -- Egress metering -----------------------------------------
// First, so the byte counter wraps `res.write` before compression
// does and therefore counts what actually goes out rather than what
@@ -1765,19 +1776,16 @@ export class PuterServer {
}
}
#prepareShutdownHooks: Promise<void> | null = null;
#prepareShutdownHooksRan = false;
/**
* Run every layer's `onServerPrepareShutdown` exactly once, whichever of
* `prepareShutdown()` / `shutdown()` gets there first; the other waits for
* that run to finish.
* `prepareShutdown()` / `shutdown()` gets there first.
*/
#runPrepareShutdownHooks(): Promise<void> {
this.#prepareShutdownHooks ??= this.#runHook(
'bottomUp',
'onServerPrepareShutdown',
);
return this.#prepareShutdownHooks;
async #runPrepareShutdownHooks() {
if (this.#prepareShutdownHooksRan) return;
this.#prepareShutdownHooksRan = true;
await this.#runHook('bottomUp', 'onServerPrepareShutdown');
}
async prepareShutdown(info: ServerDrainInfo = {}) {
+24 -81
View File
@@ -24,7 +24,6 @@ import { makeActor, type Actor } from '../../core/actor.js';
import { PuterServer } from '../../server.js';
import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
import { setupTestServer } from '../../testUtil.js';
import { testJobContext } from '../jobs/jobsTestUtil.js';
import { generateDefaultFsentries } from '../../util/userProvisioning.js';
import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
import { FULL_API_ACCESS } from '../permission/consts.js';
@@ -4049,6 +4048,7 @@ describe('AuthService (integration)', () => {
});
describe('purgeEndedSessions', () => {
const GATE_KEY = 'sessions:purge:gate';
const DAY = 24 * 60 * 60;
const makeSession = async (
@@ -4077,15 +4077,8 @@ describe('AuthService (integration)', () => {
return rows.length > 0;
};
const idOf = async (uuid: string) => {
const [row] = await server.clients.db.read(
'SELECT `id` FROM `sessions` WHERE `uuid` = ?',
[uuid],
);
return Number(row.id);
};
it('deletes rows ended past their retention and keeps the rest', async () => {
await server.clients.redis.del(GATE_KEY);
const user = await makeUser();
const id = user.id;
const oldRevoked = await makeSession(id, 'asset', 'revoked_at', 31);
@@ -4107,7 +4100,7 @@ describe('AuthService (integration)', () => {
const live = await makeSession(id, 'web');
expect(
await authService.purgeEndedSessions(testJobContext()),
await authService.purgeEndedSessions(),
).toBeGreaterThanOrEqual(3);
expect(await exists(oldRevoked)).toBe(false);
@@ -4119,84 +4112,34 @@ describe('AuthService (integration)', () => {
expect(await exists(live)).toBe(true);
});
it('starts from the saved cursor and clears it once the walk is done', async () => {
it('skips while another run holds the gate', async () => {
await server.clients.redis.del(GATE_KEY);
await server.clients.redis.set(GATE_KEY, 'other-node');
const user = await makeUser();
const below = await makeSession(user.id, 'asset', 'revoked_at', 31);
const above = await makeSession(user.id, 'asset', 'revoked_at', 31);
const ctx = testJobContext({ afterId: await idOf(below) });
const oldRevoked = await makeSession(
user.id,
'asset',
'revoked_at',
31,
);
expect(
await authService.purgeEndedSessions(ctx),
).toBeGreaterThanOrEqual(1);
expect(await exists(below)).toBe(true);
expect(await exists(above)).toBe(false);
expect(ctx.saved).toEqual([null]);
expect(await authService.purgeEndedSessions()).toBeNull();
expect(await exists(oldRevoked)).toBe(true);
await server.clients.redis.del(GATE_KEY);
});
it('clears its own checkpoints once a walk completes, so the next run starts at zero', async () => {
const SPAN = 5000;
vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
SPAN * 11,
);
const ranges: number[] = [];
const purge = vi
.spyOn(server.stores.session, 'purgeEndedInRange')
.mockImplementation(async (afterId: number) => {
ranges.push(afterId);
return 0;
});
it('frees the gate when the walk fails', async () => {
await server.clients.redis.del(GATE_KEY);
const spy = vi
.spyOn(server.clients.db, 'read')
.mockRejectedValueOnce(new Error('db down'));
try {
const first = testJobContext();
await authService.purgeEndedSessions(first);
expect(ranges).toHaveLength(11);
// Checkpointed at ten ranges, then cleared at the end.
expect(first.saved).toEqual([{ afterId: SPAN * 10 }, null]);
ranges.length = 0;
await authService.purgeEndedSessions(
testJobContext(first.saved.at(-1)),
await expect(authService.purgeEndedSessions()).rejects.toThrow(
'db down',
);
expect(ranges[0]).toBe(0);
expect(ranges).toHaveLength(11);
expect(await server.clients.redis.get(GATE_KEY)).toBeNull();
} finally {
purge.mockRestore();
vi.restoreAllMocks();
}
});
it('stops between ranges once aborted, and the next run resumes at the saved cursor', async () => {
const SPAN = 5000;
vi.spyOn(server.stores.session, 'maxId').mockResolvedValue(
SPAN * 30,
);
const ranges: number[] = [];
const first = testJobContext();
const purge = vi
.spyOn(server.stores.session, 'purgeEndedInRange')
.mockImplementation(async (afterId: number) => {
ranges.push(afterId);
// Aborted mid-walk, as a lost lease or a shutdown would.
if (ranges.length === 12) first.abort();
return 0;
});
try {
await authService.purgeEndedSessions(first);
expect(ranges).toHaveLength(12);
// Saved every 10 ranges, and where it stopped.
expect(first.saved).toEqual([
{ afterId: SPAN * 10 },
{ afterId: SPAN * 12 },
]);
ranges.length = 0;
const next = testJobContext(first.saved.at(-1));
await authService.purgeEndedSessions(next);
expect(ranges[0]).toBe(SPAN * 12);
expect(ranges.at(-1)).toBe(SPAN * 29);
expect(next.saved.at(-1)).toBeNull();
} finally {
purge.mockRestore();
vi.restoreAllMocks();
spy.mockRestore();
}
});
});
+72 -49
View File
@@ -46,7 +46,6 @@ import type { puterServices } from '../index';
import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
import type { GrantMeta } from '../permission/PermissionService.js';
import { savedAfterId, type JobRunContext } from '../jobs/JobScheduler.js';
import { PuterService } from '../types';
import type {
AccessTokenPayload,
@@ -65,14 +64,13 @@ const MAX_ORIGIN_UID_GENERATIONS = 8;
// rows double as sign-in history, so they stay longer.
const SESSION_RETENTION_SECONDS = 30 * 24 * 60 * 60;
const WEB_SESSION_RETENTION_SECONDS = 365 * 24 * 60 * 60;
const SESSION_PURGE_JOB = 'auth.sessions.purge';
const SESSION_PURGE_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
const SESSION_PURGE_LEASE_MS = 30 * 60 * 1000;
// Every node checks hourly; the gate lets one node through per week.
const SESSION_PURGE_CHECK_MS = 60 * 60 * 1000;
const SESSION_PURGE_GATE_KEY = 'sessions:purge:gate';
const SESSION_PURGE_GATE_SECONDS = 7 * 24 * 60 * 60;
// Ids per DELETE, with a pause between, so replicas keep up.
const SESSION_PURGE_SPAN = 5000;
const SESSION_PURGE_PAUSE_MS = 50;
/** Ranges walked between saves of the cursor. */
const SESSION_PURGE_SAVE_EVERY = 10;
const nowSeconds = (): number => Math.floor(Date.now() / 1000);
@@ -140,17 +138,11 @@ export interface AuthResult {
export class AuthService extends PuterService {
declare protected services: LayerInstances<typeof puterServices>;
#sessionPurge: ReturnType<typeof setInterval> | null = null;
#stopping = false;
override onServerStart(): void {
this.services.jobs.register(SESSION_PURGE_JOB, {
every: SESSION_PURGE_EVERY_MS,
scope: 'fleet',
nearPrimary: true,
leaseMs: SESSION_PURGE_LEASE_MS,
run: async (ctx) => {
const purged = await this.purgeEndedSessions(ctx);
console.log(`[auth] purged ${purged} ended sessions`);
},
});
this.#armSessionPurge();
// Users implicitly hold read access to their own email — needed for
// any permission-gated path that asks for `user:<uuid>:email:read`
@@ -172,54 +164,85 @@ export class AuthService extends PuterService {
});
}
override onServerPrepareShutdown(): void {
this.#stopping = true;
if (this.#sessionPurge) clearInterval(this.#sessionPurge);
this.#sessionPurge = null;
}
// -- Session purge -----------------------------------------------
/**
* Delete sessions revoked or expired longer ago than their retention,
* walking the table by id from the cursor an interrupted run saved. Returns
* the rows deleted; stops early once `signal` aborts.
* walking the table by id. Returns the rows deleted, or null when another
* node already holds this week's gate.
*/
async purgeEndedSessions({
signal,
state,
saveState,
}: JobRunContext): Promise<number> {
async purgeEndedSessions(): Promise<number | null> {
const claimed = await this.clients.redis.set(
SESSION_PURGE_GATE_KEY,
String(Date.now()),
'EX',
SESSION_PURGE_GATE_SECONDS,
'NX',
);
if (claimed !== 'OK') return null;
const now = nowSeconds();
const cutoffs = {
cutoff: now - SESSION_RETENTION_SECONDS,
webCutoff: now - WEB_SESSION_RETENTION_SECONDS,
};
const maxId = await this.stores.session.maxId();
let afterId = savedAfterId(state);
let purged = 0;
let unsaved = 0;
while (afterId < maxId) {
if (signal.aborted) {
// Kept for the next run if this one still holds the job.
let finished = false;
try {
const maxId = await this.stores.session.maxId();
for (
let afterId = 0;
afterId < maxId;
afterId += SESSION_PURGE_SPAN
) {
if (this.#stopping) return purged;
purged += await this.stores.session.purgeEndedInRange(
afterId,
SESSION_PURGE_SPAN,
cutoffs,
);
await new Promise((resolve) =>
setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
);
}
finished = true;
return purged;
} finally {
// A walk cut short frees the gate, so another node restarts it on
// its next check instead of next week.
if (!finished) {
try {
await saveState({ afterId });
await this.clients.redis.del(SESSION_PURGE_GATE_KEY);
} catch {
// Lost the job; the next run redoes these ranges.
// The gate expires on its own.
}
return purged;
}
purged += await this.stores.session.purgeEndedInRange(
afterId,
SESSION_PURGE_SPAN,
cutoffs,
);
afterId += SESSION_PURGE_SPAN;
if (++unsaved >= SESSION_PURGE_SAVE_EVERY) {
await saveState({ afterId });
unsaved = 0;
}
await new Promise((resolve) =>
setTimeout(resolve, SESSION_PURGE_PAUSE_MS),
);
}
// Clears any checkpoint, this run's included, so next week starts at 0.
await saveState(null);
return purged;
}
#armSessionPurge(): void {
const timer = setInterval(() => {
void this.#runSessionPurge();
}, SESSION_PURGE_CHECK_MS);
timer.unref?.();
this.#sessionPurge = timer;
}
async #runSessionPurge(): Promise<void> {
try {
const purged = await this.purgeEndedSessions();
if (purged !== null) {
console.log(`[auth] purged ${purged} ended sessions`);
}
} catch (err) {
console.warn('[auth] session purge failed', err);
}
}
// -- Public API --------------------------------------------------
@@ -2864,7 +2887,7 @@ export class AuthService extends PuterService {
#actorUserFromRow(user: UserRow) {
// Strip the password hash; pass everything else through so callers
// can read metadata, otp_enabled, etc. without
// can read metadata, desktop_bg_*, otp_enabled, etc. without
// re-fetching the user. Mirrors what /whoami exposes off of UserRow.
const { password: _password, ...rest } = user;
return {
+60 -127
View File
@@ -185,11 +185,7 @@ const aclService = () => ({
!denied.has(resource.path) && !deniedHolders.has(actor.user?.id ?? -1),
getSafeAclError: async (_actor: Actor, resource: { path: string }) =>
denied.get(resource.path) === 'forbidden'
? {
status: 403,
message: 'Forbidden',
fields: { code: 'forbidden' },
}
? { status: 403, message: 'Forbidden', fields: { code: 'forbidden' } }
: {
status: 404,
message: 'Subject does not exist',
@@ -285,8 +281,8 @@ interface ForwardedCall {
}
/**
* Each service gets its own outbox. A delivery still in flight when a test ends
* must land in that test's record, not in the next one's.
* Each service gets its own outbox. A delivery still in flight when a test
* ends must land in that test's record, not in the next one's.
*/
const buildService = (
config: IConfig,
@@ -297,14 +293,12 @@ const buildService = (
metered: MeteredLine[];
forwarded: ForwardedCall[];
eventBus: { on: ReturnType<typeof vi.fn>; emit: ReturnType<typeof vi.fn> };
jobs: { register: ReturnType<typeof vi.fn> };
} => {
const outbox: Array<{ socket?: string; envelope: DeliveryEnvelope }> = [];
const counted: DeliveryEnvelope[] = [];
const lines: MeteredLine[] = [];
const forwardedCalls: ForwardedCall[] = [];
const bus = { on: vi.fn(), emit: vi.fn() };
const jobs = { register: vi.fn() };
const built = new EventsService(
config,
{
@@ -321,7 +315,6 @@ const buildService = (
kvShareHandle: kvShareHandleStore,
} as never,
{
jobs,
eventForward: {
// A deployment with no peers has nowhere to forward to, which
// is what every test here is.
@@ -354,7 +347,10 @@ const buildService = (
acl: aclService(),
permission: permissionService(),
metering: {
bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
bufferIncrementUsages: (
actor: Actor,
usages: UsageInput[],
) => {
for (const usage of usages)
lines.push({
userUuid: actor.user?.uuid,
@@ -380,7 +376,6 @@ const buildService = (
metered: lines,
forwarded: forwardedCalls,
eventBus: bus,
jobs,
};
};
@@ -569,59 +564,6 @@ describe('the feature switch', () => {
);
});
it('hands its table sweeps to the job scheduler and arms only the pending sweep itself', () => {
const intervals = vi.spyOn(globalThis, 'setInterval');
const timeouts = vi.spyOn(globalThis, 'setTimeout');
let jobs: { register: ReturnType<typeof vi.fn> };
try {
({ jobs } = buildService({ events: { enabled: true } } as IConfig));
// The pending sweep is the region's queue, so every node runs it.
expect(intervals.mock.calls.map(([, ms]) => ms)).toEqual([10_000]);
expect(timeouts).not.toHaveBeenCalled();
} finally {
intervals.mockRestore();
timeouts.mockRestore();
}
expect(
jobs.register.mock.calls.map(([name, job]) => ({
name,
every: job.every,
scope: job.scope,
nearPrimary: job.nearPrimary ?? false,
leaseMs: job.leaseMs,
})),
).toEqual([
{
name: 'events.subscriptions.expirySweep',
every: 60 * 60_000,
scope: 'fleet',
nearPrimary: true,
leaseMs: 15 * 60_000,
},
{
name: 'events.userHandlers.gc',
every: 60 * 60_000,
scope: 'fleet',
nearPrimary: true,
leaseMs: 15 * 60_000,
},
{
name: 'events.subscriptions.creditSweep',
every: 15 * 60_000,
scope: 'fleet',
nearPrimary: false,
leaseMs: 10 * 60_000,
},
]);
});
it('registers no job when off', () => {
const { jobs } = buildService({
events: { enabled: false },
} as IConfig);
expect(jobs.register).not.toHaveBeenCalled();
});
it('spends nothing at all on a dispatch when off', async () => {
const { service: off } = buildService({
events: { enabled: false },
@@ -1264,9 +1206,7 @@ describe('matching', () => {
it('stops delivering to a durable row the moment it expires', async () => {
const { documents, file } = seedTree();
const now = Math.floor(Date.now() / 1000);
const row = (
over: Partial<DurableSubscription>,
): DurableSubscription => ({
const row = (over: Partial<DurableSubscription>): DurableSubscription => ({
subId: `durable-${seq}-${over.expiresAt}`,
token: `f#${documents.uid}`,
ownerUserId: userId,
@@ -1415,9 +1355,9 @@ describe('matching', () => {
await dispatch(closed);
await flush();
expect(
sent.map((s) => (s.envelope.event as { uid: string }).uid),
).toEqual([reachable.uid]);
expect(sent.map((s) => (s.envelope.event as { uid: string }).uid)).toEqual([
reachable.uid,
]);
});
it('publishes nothing for an event with no registry entry', async () => {
@@ -1459,9 +1399,9 @@ describe('matching', () => {
await flush();
expect(sent).toHaveLength(1);
expect((sent[0].envelope.event as { path: string }).path).toBe(
inside.path,
);
expect(
(sent[0].envelope.event as { path: string }).path,
).toBe(inside.path);
});
it('does not reach a sibling that only shares the missing name as a prefix', async () => {
@@ -1533,9 +1473,8 @@ describe('coalescing', () => {
await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
expect(sent).toHaveLength(2);
expect(
sent.map((s) => (s.envelope.event as { uid: string }).uid).sort(),
).toEqual([file.uid, second.uid].sort());
expect(sent.map((s) => (s.envelope.event as { uid: string }).uid).sort())
.toEqual([file.uid, second.uid].sort());
});
it('counts exactly what it delivered', async () => {
@@ -1681,9 +1620,9 @@ describe('limits', () => {
}
}
await expect(subscribe(`fs:${documents.uid}`)).resolves.toMatchObject({
subId: expect.any(String),
});
await expect(subscribe(`fs:${documents.uid}`)).resolves.toMatchObject(
{ subId: expect.any(String) },
);
});
it('refuses unsubscribes past their own per-minute budget', async () => {
@@ -1752,7 +1691,9 @@ describe('limits', () => {
),
);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(
1,
);
expect(results.filter((r) => r.status === 'rejected')).toHaveLength(
attempts - 1,
);
@@ -1949,9 +1890,9 @@ describe('limits', () => {
).toHaveLength(EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT);
// 200 evaluated, 20 never reached: everything past the delivery cap is
// told it missed something, itself capped.
expect(sent.filter((s) => s.envelope.event.op === 'gap')).toHaveLength(
EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT,
);
expect(
sent.filter((s) => s.envelope.event.op === 'gap'),
).toHaveLength(EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT);
expect(
sent.find((s) => s.envelope.event.op === 'gap')?.envelope.event,
).toMatchObject({ reason: 'filter_evaluation_limit' });
@@ -2002,7 +1943,8 @@ describe('the socket surface', () => {
fire: (event, ...args) =>
(
handlers.get(event) as
((...a: unknown[]) => void) | undefined
| ((...a: unknown[]) => void)
| undefined
)?.(...args),
};
};
@@ -2183,14 +2125,10 @@ it('names the delivery channel the clients listen on', () => {
describe('asking a kv subscription for the value', () => {
it('is recorded on the row and reported in the view', async () => {
const { sub } = await service.subscribe(
appActorFor(OWN_APP),
socketId,
{
subject: `kv:${OWN_APP}:cart`,
includeValue: true,
},
);
const { sub } = await service.subscribe(appActorFor(OWN_APP), socketId, {
subject: `kv:${OWN_APP}:cart`,
includeValue: true,
});
expect(sub.includeValue).toBe(true);
const plain = await subscribeKv(`kv:${OWN_APP}:cart`);
@@ -2273,7 +2211,10 @@ describe('resolving a kv subject', () => {
});
it('refuses a pattern the grammar cannot enumerate from a key', async () => {
for (const subject of [`kv:${OWN_APP}:ca*rt`, `kv:${OWN_APP}:cart?`])
for (const subject of [
`kv:${OWN_APP}:ca*rt`,
`kv:${OWN_APP}:cart?`,
])
await expect(subscribeKv(subject)).rejects.toSatisfy(
(err: unknown) =>
isHttpError(err) && err.legacyCode === 'invalid_kv_pattern',
@@ -2388,7 +2329,8 @@ describe('delivering a kv change', () => {
const eventFor = (subId: string) =>
sent.find((one) => one.envelope.subId === subId)?.envelope.event as
Record<string, unknown> | undefined;
| Record<string, unknown>
| undefined;
it('hands the value to the row that asked for it and to no other', async () => {
vi.useFakeTimers();
@@ -2503,9 +2445,7 @@ describe('delivering a kv change', () => {
await dispatchKv(['cart'], { values: ['kept'] });
await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
const deliveries = sent.filter(
(one) => one.envelope.event.op === 'set',
);
const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
const gaps = sent.filter((one) => one.envelope.event.op === 'gap');
expect(deliveries).toHaveLength(128);
expect(gaps).toHaveLength(1);
@@ -2552,12 +2492,12 @@ describe('delivering a kv change', () => {
await dispatchKv(['cart']);
await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
expect(
sent.filter((one) => one.envelope.event.op === 'set'),
).toHaveLength(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit);
expect(
sent.filter((one) => one.envelope.event.op === 'gap'),
).toHaveLength(1);
expect(sent.filter((one) => one.envelope.event.op === 'set')).toHaveLength(
EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.limit,
);
expect(sent.filter((one) => one.envelope.event.op === 'gap')).toHaveLength(
1,
);
});
it('uses the free KV limit when owner policy lookup fails', async () => {
@@ -2568,9 +2508,7 @@ describe('delivering a kv change', () => {
await dispatchKv(['cart']);
await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
expect(
sent.filter((one) => one.envelope.event.op === 'set'),
).toHaveLength(
expect(sent.filter((one) => one.envelope.event.op === 'set')).toHaveLength(
EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT.bySubscription[
DEFAULT_FREE_SUBSCRIPTION
],
@@ -2583,9 +2521,7 @@ describe('delivering a kv change', () => {
await seedKvSubscriptions(
EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT.bySubscription[
DEFAULT_FREE_SUBSCRIPTION
] -
128 +
1,
] - 128 + 1,
{ includeValue: true, match: 'elsewhere', tag: 'filtered-out' },
);
@@ -2607,9 +2543,7 @@ describe('delivering a kv change', () => {
});
await vi.advanceTimersByTimeAsync(EVENTS_COALESCE_WINDOW_MS + 1);
const deliveries = sent.filter(
(one) => one.envelope.event.op === 'set',
);
const deliveries = sent.filter((one) => one.envelope.event.op === 'set');
expect(deliveries).toHaveLength(128);
expect(
deliveries.every(
@@ -2835,7 +2769,8 @@ describe('what a kv dispatch costs', () => {
});
describe('the cross-app kv gate', () => {
const grantRead = (app: string) => grants.add(`app-data:${app}:kv:read`);
const grantRead = (app: string) =>
grants.add(`app-data:${app}:kv:read`);
const crossAppService = () =>
buildService({
@@ -2899,8 +2834,7 @@ describe('the cross-app kv gate', () => {
({ service } = crossAppService());
await expect(subscribeKv(`kv:${OTHER_APP}:cart`)).rejects.toSatisfy(
(err: unknown) =>
isHttpError(err) && err.legacyCode === 'forbidden',
(err: unknown) => isHttpError(err) && err.legacyCode === 'forbidden',
);
});
@@ -2911,7 +2845,8 @@ describe('the cross-app kv gate', () => {
await expect(subscribeKv('kv:app-gone:cart')).rejects.toSatisfy(
(err: unknown) =>
isHttpError(err) && err.legacyCode === 'subject_does_not_exist',
isHttpError(err) &&
err.legacyCode === 'subject_does_not_exist',
);
});
@@ -2925,8 +2860,7 @@ describe('the cross-app kv gate', () => {
});
await expect(subscribeKv(`kv:${OTHER_APP}:cart`)).rejects.toSatisfy(
(err: unknown) =>
isHttpError(err) && err.legacyCode === 'forbidden',
(err: unknown) => isHttpError(err) && err.legacyCode === 'forbidden',
);
});
@@ -3201,7 +3135,9 @@ describe('cross-user kv handles', () => {
expect(off.kvHandlesEnabled).toBe(false);
expect(service.kvHandlesEnabled).toBe(true);
await expect(subscribeAsGuest(`kv:${handle}:*`, off)).rejects.toSatisfy(
await expect(
subscribeAsGuest(`kv:${handle}:*`, off),
).rejects.toSatisfy(
(err: unknown) =>
isHttpError(err) &&
err.legacyCode === 'events_kv_handles_disabled',
@@ -3409,13 +3345,10 @@ describe('cross-user kv handles', () => {
it.each([
['an unknown handle', () => undefined],
['a revoked handle', () => mintHandle({ revokedAt: 1 })],
[
'a handle whose grant the caller does not hold',
() => {
mintHandle();
grants.delete(permission);
},
],
['a handle whose grant the caller does not hold', () => {
mintHandle();
grants.delete(permission);
}],
])('answers %s as absent', async (_case, setUp) => {
setUp();
await expect(subscribeAsGuest(`kv:${handle}:*`)).rejects.toSatisfy(
File diff suppressed because it is too large. Load diff
@@ -19,29 +19,19 @@
/**
* What deleting an app leaves behind on the events side: its durable rows,
* their backlog and its published handlers can never deliver again — the grant
* identity they are re-checked under no longer resolves — so `settleDeletedApp`
* tears them down on the `app.changed` bus key rather than leaving them to cost
* their holder an anchor slot forever. Also the hourly sweep that catches
* whatever neither this nor `workers.destroy` reached.
* their backlog and its published handlers can never deliver again — the
* grant identity they are re-checked under no longer resolves — so
* `settleDeletedApp` tears them down on the `app.changed` bus key rather than
* leaving them to cost their holder an anchor slot forever. Also the hourly
* sweep that catches whatever neither this nor `workers.destroy` reached.
*/
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { v4 as uuidv4 } from 'uuid';
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
import { testJobContext } from '../jobs/jobsTestUtil.js';
import type { IConfig } from '../../types.js';
import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
import { EVENTS_WORKER_SESSION_NAME } from './workerRuntime.js';
import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
const BOOT_TIMEOUT_MS = 120_000;
@@ -135,13 +125,10 @@ const emitDeleted = async (appUid: string): Promise<void> => {
const publish = (
token: string,
body: { appUid: string; name: string; source: string },
): Promise<ApiResponse> =>
call('POST', '/events/handlers/publish', token, body);
): Promise<ApiResponse> => call('POST', '/events/handlers/publish', token, body);
/**
* A durable `single` subscription bound to a published handler, with the grants
* a background delivery needs. Returns the subscribed subId.
*/
/** A durable `single` subscription bound to a published handler, with the
* grants a background delivery needs. Returns the subscribed subId. */
const subscribeBackgroundHandler = async (
app: { uid: string; token: string },
handlerName: string,
@@ -175,87 +162,6 @@ const subscribeBackgroundHandler = async (
return subscribed.body.subId as string;
};
describe('user scopes of a deleted app or account', () => {
const publishUserHandler = (
appUid: string,
holder: number,
name = 'mine',
) =>
env.server.stores.eventHandler.publish({
scope: { appUid, holder },
name,
source: SOURCE,
broadcast: false,
});
const userHandlers = (appUid: string, holder: number) =>
env.server.stores.eventHandler.listForApp({ appUid, holder });
let destroyed: Array<{ appUid: string; scope: string; userId: number }>;
beforeAll(() => {
env.server.clients.event.on('events.worker.destroy', (_key, event) => {
destroyed?.push({
appUid: event.appUid,
scope: event.scope,
userId: event.actor.user.id,
});
});
});
beforeEach(async () => {
destroyed = [];
await env.server.clients.db.write(
'DELETE FROM `event_user_handlers`',
[],
);
});
it('removes every holder`s scope with the app, announcing each as the holder', async () => {
const app = await makeApp(userId);
await publishUserHandler(app.uid, userId);
await publishUserHandler(app.uid, otherUserId);
await publishUserHandler(app.uid, otherUserId, 'second');
const untouched = await makeApp(userId);
await publishUserHandler(untouched.uid, userId);
await emitDeleted(app.uid);
expect(await userHandlers(app.uid, userId)).toEqual([]);
expect(await userHandlers(app.uid, otherUserId)).toEqual([]);
expect(
destroyed
.filter((event) => event.scope === 'user')
.map((event) => [event.appUid, event.userId])
.sort(),
).toEqual(
[
[app.uid, userId],
[app.uid, otherUserId],
].sort(),
);
// Another app's scope is not touched.
expect(await userHandlers(untouched.uid, userId)).toHaveLength(1);
});
it('removes a deleted account`s scopes in every app, and no one else`s', async () => {
const first = await makeApp(userId);
const second = await makeApp(userId);
for (const app of [first, second]) {
await publishUserHandler(app.uid, otherUserId);
await publishUserHandler(app.uid, userId);
}
await env.server.clients.event.emitAndWait(
'user.delete',
{ user_id: otherUserId, appUids: [] },
{},
);
for (const app of [first, second]) {
expect(await userHandlers(app.uid, otherUserId)).toEqual([]);
expect(await userHandlers(app.uid, userId)).toHaveLength(1);
}
});
});
describe('an app deleted while it has events state', () => {
it('clears the rows, the backlog and the handlers when the app is deleted', async () => {
const app = await makeApp(userId);
@@ -280,17 +186,15 @@ describe('an app deleted while it has events state', () => {
await emitDeleted(app.uid);
const listed =
await env.server.stores.durableSubscription.listForHolder(userId, {
appUid: app.uid,
});
const listed = await env.server.stores.durableSubscription.listForHolder(
userId,
{ appUid: app.uid },
);
expect(listed.items).toEqual([]);
expect(await env.server.stores.pendingDelivery.depth(subId)).toBe(0);
expect(
await env.server.stores.eventHandler.listForApp(
appHandlerScope(app.uid),
),
).toEqual([]);
expect(await env.server.stores.eventHandler.listForApp(app.uid)).toEqual(
[],
);
});
it("clears the rows and handlers of a deleted account's apps, and the account's subscription keys", async () => {
@@ -313,16 +217,14 @@ describe('an app deleted while it has events state', () => {
{},
);
const listed =
await env.server.stores.durableSubscription.listForHolder(userId, {
appUid: app.uid,
});
const listed = await env.server.stores.durableSubscription.listForHolder(
userId,
{ appUid: app.uid },
);
expect(listed.items).toEqual([]);
expect(
await env.server.stores.eventHandler.listForApp(
appHandlerScope(app.uid),
),
).toEqual([]);
expect(await env.server.stores.eventHandler.listForApp(app.uid)).toEqual(
[],
);
expect(
await env.server.stores.eventSubscription.isRegionWarm(goneUserId),
).toBe(false);
@@ -357,10 +259,12 @@ describe('an app deleted while it has events state', () => {
expiresAt: null,
});
const { actor: ownerActor } =
await env.server.services.auth.authenticate(env.users.user.token);
const { actor: otherActor } =
await env.server.services.auth.authenticate(env.users.other.token);
const { actor: ownerActor } = await env.server.services.auth.authenticate(
env.users.user.token,
);
const { actor: otherActor } = await env.server.services.auth.authenticate(
env.users.other.token,
);
const ownerToken = await env.server.services.auth.createWorkerAppToken(
ownerActor!,
app.uid,
@@ -423,9 +327,7 @@ describe('an app deleted while it has events state', () => {
await env.server.stores.app.delete((row as { id: number }).id);
const revoked =
await env.server.services.events.sweepStrandedWorkerSessions(
testJobContext(),
);
await env.server.services.events.sweepStrandedWorkerSessions();
expect(revoked).toBeGreaterThanOrEqual(1);
// `authenticate()` can no longer resolve the token at all — the app
@@ -464,60 +366,9 @@ describe('an app deleted while it has events state', () => {
EVENTS_WORKER_SESSION_NAME,
);
await env.server.services.events.sweepStrandedWorkerSessions(
testJobContext(),
);
await env.server.services.events.sweepStrandedWorkerSessions();
const reauth = await env.server.services.auth.authenticate(token);
expect(reauth).not.toHaveProperty('reauth');
});
it('reaches a stranded session past one run`s two pages on the next run, then wraps', async () => {
const db = env.server.clients.db;
const now = Math.floor(Date.now() / 1000);
const meta = JSON.stringify({
worker_name: EVENTS_WORKER_SESSION_NAME,
});
const workerSession = (appUid: string) => ({
statement:
'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`, `kind`, `app_uid`) ' +
"VALUES (?, ?, ?, ?, ?, 'worker', ?)",
values: [uuidv4(), userId, meta, now, now, appUid],
});
// A run reads two pages of 500; these fill them, all still standing.
const live = Array.from(
{ length: 1000 },
(_, i) => `app-live-${uuidv4().slice(0, 8)}-${i}`,
);
await db.batchWrite(
live.map((uid) => ({
statement:
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
values: [uid, uid, uid, `https://${uid}.example/`, userId],
})),
);
await db.batchWrite(live.map(workerSession));
const stranded = workerSession(`app-gone-${uuidv4()}`);
await db.batchWrite([stranded]);
const strandedRevoked = async () => {
const [row] = await db.pread(
'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
[stranded.values[0]],
);
return row.revoked_at !== null;
};
const first = testJobContext();
await env.server.services.events.sweepStrandedWorkerSessions(first);
expect(await strandedRevoked()).toBe(false);
expect(first.saved.at(-1)).toMatchObject({
afterId: expect.any(Number),
});
const second = testJobContext(first.saved.at(-1));
await env.server.services.events.sweepStrandedWorkerSessions(second);
expect(await strandedRevoked()).toBe(true);
// The end of the table wraps the cursor back to the start.
expect(second.saved.at(-1)).toBeNull();
});
});
@@ -872,71 +872,6 @@ describe('what a write costs once a durable row exists', () => {
expect(deliveryOf(path)).toBeUndefined();
});
it('publishes the expiry reap`s bump, so a region still caching the row drops it', async () => {
await clearRows();
const created = await subscribe(env.users.user.token, {
expiresAt: Math.floor(Date.now() / 1000) + 3600,
});
const subId = created.body.subId as string;
const stale =
await env.server.stores.durableSubscription.getBySubId(subId);
await env.server.clients.db.write(
`UPDATE \`${TABLE}\` SET \`expires_at\` = ? WHERE \`sub_id\` = ?`,
[Math.floor(Date.now() / 1000) - 1, subId],
);
const heard: Array<Record<string, unknown>> = [];
const listener = (_key: string, data: unknown) => {
heard.push(data as Record<string, unknown>);
};
const announced = vi.spyOn(
env.server.services.eventForward,
'announceGeneration',
);
env.server.clients.event.on(
'outer.pubsub.events.generationBumped',
listener as never,
);
try {
await expect(events().sweepExpired()).resolves.toBe(1);
// Over the addressed peer channel as well as the bus.
expect(announced).toHaveBeenCalledWith(
expect.objectContaining({ userId }),
true,
'subscription',
);
} finally {
env.server.clients.event.off(
'outer.pubsub.events.generationBumped',
listener as never,
);
announced.mockRestore();
}
const published = heard.find(
(bump) => bump.userId === userId && bump.durable === true,
);
expect(published).toBeDefined();
// Another region's Redis still holds the row; this one stands in for it.
await env.server.stores.eventSubscription.cacheDurable([stale!]);
delivered.length = 0;
const before = `${anchor}/stale-${uuidv4().slice(0, 8)}.txt`;
await fs().touch(userId, { path: before });
await settle(before);
expect(deliveryOf(before)?.subId).toBe(subId);
await env.server.clients.event.emitAndWait(
'outer.pubsub.events.generationBumped',
published!,
{ from_outside: true },
);
delivered.length = 0;
const after = `${anchor}/peer-swept-${uuidv4().slice(0, 8)}.txt`;
await fs().touch(userId, { path: after });
await quiet();
expect(deliveryOf(after)).toBeUndefined();
});
});
describe('where a durable delivery is addressed', () => {
@@ -616,210 +616,6 @@ describe('reading again from a saved cursor', () => {
});
});
describe('starting from the end', () => {
const SUBJECT = 'notif:app-user';
const message = async (title: string, inApp = appUid): Promise<string> =>
(
await env.server.stores.notification.create({
userId,
value: { type: 'app.message', title },
type: 'app.message',
audience: 'app-user',
appUid: inApp,
group: `group-${uuidv4()}`,
})
).uid as string;
const fromEnd = async (token = appToken): Promise<string> => {
const end = await fetchPage(token, { subject: SUBJECT, from: 'end' });
expect(end.status).toBe(200);
expect(end.body.items).toEqual([]);
expect(end.body.cursor).toBeTruthy();
return end.body.cursor!;
};
it('answers a cursor on an empty slice that returns what comes after', async () => {
await clearMailboxes();
const cursor = await fromEnd();
const later = await message('later');
const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
expect(uidsOf(next)).toEqual([later]);
});
it('skips what is there and returns only newer rows', async () => {
await clearMailboxes();
for (let i = 0; i < 3; i++) await message(`before ${i}`);
const cursor = await fromEnd();
const quiet = await fetchPage(appToken, { subject: SUBJECT, cursor });
expect(quiet.body.items).toEqual([]);
expect(quiet.body.cursor).toBeUndefined();
const later = [await message('after 0'), await message('after 1')];
const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
expect(uidsOf(next)).toEqual(later);
});
it('returns a row replaced after the cursor was taken', async () => {
await clearMailboxes();
const old = await message('old');
await message('older still unchanged');
const cursor = await fromEnd();
await env.server.services.notification.notifyUpdate(
old,
userId,
{ title: 'replaced' },
{ type: 'app.message', replace: true },
);
const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
expect(uidsOf(next)).toEqual([old]);
expect(next.body.items![0]).toMatchObject({
replaced: true,
notification: { title: 'replaced' },
});
});
it('returns a replacement stamped by a clock behind the server`s', async () => {
await clearMailboxes();
const behind = Date.now() - 60_000;
const [early, late] = [await message('early'), await message('late')];
const replace = (uid: string, title: string, replacedAt: number) =>
env.server.stores.notification.updateValue(
uid,
userId,
{ type: 'app.message', title },
{ onlyUnseen: true, replacedAt },
);
await replace(early, 'replaced before', behind - 1_000);
const cursor = await fromEnd();
await replace(late, 'replaced after', behind);
const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
expect(uidsOf(next)).toEqual([late]);
expect(next.body.items![0]).toMatchObject({
notification: { title: 'replaced after' },
});
});
it('keeps to the slice the subject names', async () => {
await clearMailboxes();
await message('own before');
const cursor = await fromEnd();
await message('other app', otherAppUid);
await seed(userId, { title: 'account row' });
const own = await message('own after');
const next = await fetchPage(appToken, { subject: SUBJECT, cursor });
expect(uidsOf(next)).toEqual([own]);
});
it('answers no cursor for a slice the caller cannot see', async () => {
await clearMailboxes();
await message('other app', otherAppUid);
for (const [token, subject] of [
[appToken, `notif:${otherAppUid}:app-user`],
[appToken, `notif:${userUuid}:account`],
[appReadUrlToken, SUBJECT],
] as const) {
const page = await fetchPage(token, { subject, from: 'end' });
expect(page.status).toBe(200);
expect(page.body.items).toEqual([]);
expect(page.body.cursor).toBeUndefined();
}
});
it('reads from the cursor when one is given', async () => {
await clearMailboxes();
const uids = [await message('m0'), await message('m1')];
const first = await fetchPage(appToken, { subject: SUBJECT, limit: 1 });
const page = await fetchPage(appToken, {
subject: SUBJECT,
cursor: first.body.cursor,
from: 'end',
});
expect(uidsOf(page)).toEqual([uids[1]]);
});
it('rejects a `from` that is neither start nor end', async () => {
const page = await fetchPage(appToken, {
subject: SUBJECT,
from: 'middle',
});
expect(page.status).toBe(400);
const start = await fetchPage(appToken, {
subject: SUBJECT,
from: 'start',
});
expect(start.status).toBe(200);
});
it('starts the SDK feed without catch-up in one read', async () => {
await clearMailboxes();
for (let i = 0; i < 3; i++) await message(`before ${i}`);
const appActor = (await env.server.services.auth.authenticate(appToken))
.actor!;
const reads: Array<Record<string, unknown>> = [];
let receive: (delivery: { event: unknown }) => void = () => {};
const titles: string[] = [];
const feed = new NotificationFeed(
{
authToken: appToken,
events: {
onLocal: async (
_subject: string,
handler: typeof receive,
) => {
receive = handler;
return { off: async () => {} };
},
fetch: async (options: Record<string, unknown>) => {
reads.push(options);
return env.server.services.events.fetchMissed(
appActor,
options,
);
},
},
} as never,
(notification: { title: string | null }) => {
titles.push(String(notification.title));
},
{ catchUp: false },
);
try {
await feed.start();
expect(reads).toEqual([
{ subject: SUBJECT, limit: 200, from: 'end' },
]);
await message('after start');
receive({ event: { op: 'gap', reason: 'reconnect' } });
await vi.waitFor(() => expect(reads).toHaveLength(2));
await new Promise((resolve) => setTimeout(resolve, 0));
} finally {
await feed.off();
}
expect(reads[1]).toEqual({
subject: SUBJECT,
limit: 200,
cursor: expect.any(String),
});
expect(titles).toEqual(['after start']);
});
});
describe('cross-user isolation', () => {
it('returns nothing of another account, whatever the subject names', async () => {
await clearMailboxes();
@@ -79,14 +79,6 @@ vi.mock(import('@opentelemetry/api'), async (importOriginal) => ({
metricCalls.push({ name, value, attributes });
},
}),
createHistogram: (name: string) => ({
record: (
value: number,
attributes: Record<string, unknown> = {},
) => {
metricCalls.push({ name, value, attributes });
},
}),
}),
},
}));
@@ -449,7 +441,6 @@ const makeRegion = (
): Promise<{
row: DurableSubscription;
bump: { userId: number; generation: number };
reaped: [];
}> => {
const row = durableRow({
holderUserId: input.holderUserId as number,
@@ -478,7 +469,6 @@ const makeRegion = (
row.ownerUserId,
),
},
reaped: [],
};
},
},
@@ -45,10 +45,7 @@ import {
EVENTS_WORKER_SOURCE_MAX_BYTES,
} from '../../controllers/events/limits.js';
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
import {
appHandlerScope,
hashContent,
} from '../../stores/events/EventHandlerStore.js';
import { hashContent } from '../../stores/events/EventHandlerStore.js';
import type { IConfig } from '../../types.js';
import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
import type { DurableSubscriptionView } from './EventsService.js';
@@ -223,8 +220,7 @@ afterAll(async () => {
beforeEach(async () => {
await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
for (const row of (await durable().listActiveForHolderPage(userId, null, 0))
.rows)
for (const row of await durable().listActiveForHolder(userId, null))
await durable().remove(row);
await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
await env.server.stores.eventSubscription.rebuildDurable(userId, []);
@@ -535,7 +531,7 @@ describe('what a publish reads', () => {
if (sql.includes(BY_NAME_SQL) && !landed) {
landed = true;
await env.server.stores.eventHandler.publish({
scope: appHandlerScope(appUid),
appUid,
name: 'ingestUpload',
source: NEXT_SOURCE,
replace: true,
@@ -593,7 +589,7 @@ describe('what a publish reads', () => {
pread.mockRestore();
}
const standing = await env.server.stores.eventHandler.getByName(
appHandlerScope(appUid),
appUid,
'ingestUpload',
);
expect(standing?.sourceHash).toBe(hashContent(THIRD_SOURCE));
@@ -606,7 +602,7 @@ describe('what a publish reads', () => {
if (sql.includes(BY_NAME_SQL) && !beaten) {
beaten = true;
await env.server.stores.eventHandler.publish({
scope: appHandlerScope(appUid),
appUid,
name: 'ingestUpload',
source: THIRD_SOURCE,
});
@@ -662,7 +658,7 @@ describe('what a publish reads', () => {
pread.mockRestore();
}
await expect(
env.server.stores.eventHandler.countForApp(appHandlerScope(appUid)),
env.server.stores.eventHandler.countForApp(appUid),
).resolves.toBe(0);
});
@@ -781,9 +777,7 @@ describe('binding a subscription to a handler', () => {
expect(refused.status).toBe(404);
expect(refused.body.code).toBe('events_handler_not_found');
expect(
(await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
).toEqual([]);
expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
});
it('binds an inline body whose hash is what is published', async () => {
@@ -811,9 +805,7 @@ describe('binding a subscription to a handler', () => {
expect(refused.status).toBe(409);
expect(refused.body.code).toBe('events_handler_hash_mismatch');
expect(
(await durable().listActiveForHolderPage(userId, appUid, 0)).rows,
).toEqual([]);
expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]);
});
it('still needs a name for a subscription owed to one consumer', async () => {
@@ -383,7 +383,7 @@ describe('revoking a handle', () => {
const sub = await subscribeDurable(handle);
const store = env.server.stores.durableSubscription;
const byHolder = vi.spyOn(store, 'listActiveForHolderPage');
const byHolder = vi.spyOn(store, 'listActiveForHolder');
const settle = vi.spyOn(events(), 'settleRevokedGrant');
try {
@@ -400,9 +400,7 @@ describe('revoking a handle', () => {
for (const call of byHolder.mock.calls)
expect(call.slice(1)).toEqual([
null,
0,
undefined,
{ includeSuspended: true, excludeSchedules: true },
{ includeSuspended: true },
]);
} finally {
byHolder.mockRestore();
@@ -444,9 +442,7 @@ describe('revoking a handle', () => {
const page = await shares().listKvHandles(owner.actor, {
limit: 100,
});
const deeperRow = page.items.find(
(one) => one.handle === deeper.handle,
);
const deeperRow = page.items.find((one) => one.handle === deeper.handle);
expect(deeperRow?.revokedAt).toBeTypeOf('number');
expect((await countLive()).total).toBe(before.total);
});
+29 -153
View File
@@ -36,17 +36,13 @@ import {
DurableQuotaRaceLost,
type DurableSubscriptionInput,
} from '../../stores/events/DurableSubscriptionStore.js';
import {
scheduleToken,
type DurableSubscription,
} from '../../stores/events/types.js';
import type { DurableSubscription } from '../../stores/events/types.js';
import type { FSEntry } from '../../stores/fs/FSEntry.js';
import {
DEFAULT_FREE_SUBSCRIPTION,
DEFAULT_TEMP_SUBSCRIPTION,
} from '../metering/consts.js';
import type { UsageInput } from '../metering/types.js';
import { testJobContext } from '../jobs/jobsTestUtil.js';
import type { IConfig } from '../../types.js';
import { EVENTS_COSTS } from './costs.js';
import { EventsService } from './EventsService.js';
@@ -66,12 +62,6 @@ let solvent: Set<number>;
/** Set to have the next create lose the last slot after its insert. */
let raceLost: boolean;
let emitted: Array<{ key: string; data: unknown }>;
/** Rows one credit-sweep page takes; null for the service's own batch. */
let pageSize: number | null;
/** Rows one holder page takes; null for all of them in one page. */
let holderPageSize: number | null;
/** What `services.schedules` hears about changed schedule rows. */
let schedulesChanged: (rows: readonly DurableSubscription[]) => void;
const anchorUid = (): string => `docs-${seq}`;
const anchorPath = (): string => `/u${userId}/Documents`;
@@ -109,10 +99,7 @@ const actorFor = (over: Partial<Actor> = {}): Actor =>
}) as unknown as Actor;
const appActor = (appUid: string): Actor =>
actorFor({
app: { uid: appUid, id: 1 },
effectiveApp: { uid: appUid, id: 1 },
});
actorFor({ app: { uid: appUid, id: 1 }, effectiveApp: { uid: appUid, id: 1 } });
const anchorEntry = (): FSEntry =>
({
@@ -137,9 +124,6 @@ beforeEach(() => {
solvent = new Set([userId]);
raceLost = false;
emitted = [];
pageSize = null;
holderPageSize = null;
schedulesChanged = () => undefined;
service = new EventsService(
{ events: { enabled: true } } as IConfig,
@@ -171,49 +155,26 @@ beforeEach(() => {
return {
row,
bump: { userId: input.ownerUserId, generation: 1 },
reaped: [],
};
},
// Keyset paging over `rows`, a row's id being its position + 1.
listSuspendedPage: async (
reason: string,
afterId: number,
batchSize: number,
) => {
const size = pageSize ?? batchSize;
const page = rows
.map((row, i) => ({ row, id: i + 1 }))
.filter(
({ row, id }) =>
id > afterId && row.suspendedReason === reason,
)
.slice(0, size);
return {
rows: page.map(({ row }) => row),
nextId: page.length < size ? null : page.at(-1)!.id,
};
},
// Keyset paging over `rows`, a row's id being its position + 1.
listSuspendedForHolderPage: async (
listSuspendedPage: async (reason: string, afterId: number) =>
afterId === 0
? {
rows: rows.filter(
(row) => row.suspendedReason === reason,
),
nextId: null,
}
: { rows: [], nextId: null },
listSuspendedForHolder: async (
holderUserId: number,
reason: string,
afterId: number,
) => {
const page = rows
.map((row, i) => ({ row, id: i + 1 }))
.filter(
({ row, id }) =>
id > afterId &&
row.holderUserId === holderUserId &&
row.suspendedReason === reason,
);
const size = holderPageSize ?? page.length;
const taken = page.slice(0, size);
return {
rows: taken.map(({ row }) => row),
nextId: page.length > size ? taken.at(-1)!.id : null,
};
},
) =>
rows.filter(
(row) =>
row.holderUserId === holderUserId &&
row.suspendedReason === reason,
),
resume: async (resuming: readonly DurableSubscription[]) => {
for (const row of resuming) {
resumed.push(row.subId);
@@ -272,7 +233,10 @@ beforeEach(() => {
// plan allows, not about who agreed to it.
permission: { check: async () => true },
metering: {
bufferIncrementUsages: (actor: Actor, usages: UsageInput[]) => {
bufferIncrementUsages: (
actor: Actor,
usages: UsageInput[],
) => {
for (const usage of usages)
metered.push({
userUuid: actor.user?.uuid,
@@ -283,11 +247,6 @@ beforeEach(() => {
solvent.has(Number(actor.user?.id ?? -1)),
getActorSubscription: async () => ({ id: plan }),
},
schedules: {
onSubscriptionsChanged: async (
changed: readonly DurableSubscription[],
) => schedulesChanged(changed),
},
} as never,
);
});
@@ -299,105 +258,23 @@ describe('a suspension waiting on a balance', () => {
suspendedReason: 'no_credit',
});
it('comes back once the account has budget again, and every region hears it', async () => {
it('comes back once the account has budget again', async () => {
const row = suspended();
rows.push(row);
await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(1);
await expect(service.sweepNoCredit()).resolves.toBe(1);
expect(resumed).toEqual([row.subId]);
expect(row.suspendedReason).toBeNull();
expect(emitted).toContainEqual({
key: 'outer.pubsub.events.generationBumped',
data: { userId, generation: 1, durable: true },
});
});
it('stays out of service while the account still has none', async () => {
rows.push(suspended());
solvent.clear();
await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
await expect(service.sweepNoCredit()).resolves.toBe(0);
expect(resumed).toEqual([]);
});
it('reaches rows past one run`s page cap across runs, then wraps', async () => {
// One row per page, so a run's 200 pages stop at 200 rows.
pageSize = 1;
for (let i = 0; i < 450; i++) rows.push(suspended());
for (const [i, row] of rows.entries()) row.holderUserId = 10_000 + i;
// Only the last holder has a balance again.
solvent = new Set([10_000 + 449]);
const first = testJobContext();
await expect(service.sweepNoCredit(first)).resolves.toBe(0);
expect(first.saved.at(-1)).toEqual({ afterId: 200 });
const second = testJobContext(first.saved.at(-1));
await expect(service.sweepNoCredit(second)).resolves.toBe(0);
expect(second.saved.at(-1)).toEqual({ afterId: 400 });
const third = testJobContext(second.saved.at(-1));
await expect(service.sweepNoCredit(third)).resolves.toBe(1);
expect(resumed).toEqual([rows[449].subId]);
// The end of the table wraps the cursor back to the start.
expect(third.saved.at(-1)).toBeNull();
});
it('stops between pages once its signal aborts, keeping the last full page', async () => {
pageSize = 1;
for (let i = 0; i < 5; i++) rows.push(suspended());
for (const [i, row] of rows.entries()) row.holderUserId = 20_000 + i;
const ctx = testJobContext();
const save = ctx.saveState;
ctx.saveState = async (next) => {
await save(next);
if (ctx.saved.length === 2) ctx.abort();
};
await service.sweepNoCredit(ctx);
expect(ctx.saved).toEqual([{ afterId: 1 }, { afterId: 2 }]);
});
it('resumes no later holder page once its signal aborts inside a page`s hook, keeping the checkpoint', async () => {
// A full sweep page, so finishing it would move the checkpoint.
pageSize = 3;
holderPageSize = 1;
for (let i = 0; i < 3; i++)
rows.push(
durableRow({
subId: `sched-${i}`,
token: scheduleToken(`schedule-${seq}-${i}`),
suspendedAt: Math.floor(Date.now() / 1000),
suspendedReason: 'no_credit',
}),
);
const ctx = testJobContext();
const heard: string[][] = [];
schedulesChanged = (changed) => {
heard.push(changed.map((row) => row.subId));
ctx.abort();
};
await expect(service.sweepNoCredit(ctx)).resolves.toBe(1);
expect(resumed).toEqual(['sched-0']);
expect(heard).toEqual([['sched-0']]);
expect(rows[1].suspendedReason).toBe('no_credit');
expect(ctx.saved).toEqual([]);
});
it('wraps a saved cursor past the end of a shrunken table back to the start', async () => {
rows.push(suspended());
const ctx = testJobContext({ afterId: 999 });
await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
expect(ctx.saved).toEqual([null]);
});
it('writes no cursor while there is nothing to page through', async () => {
const ctx = testJobContext();
await expect(service.sweepNoCredit(ctx)).resolves.toBe(0);
expect(ctx.saved).toEqual([]);
});
it('leaves a suspension it did not cause alone', async () => {
rows.push(
durableRow({
@@ -406,7 +283,7 @@ describe('a suspension waiting on a balance', () => {
}),
);
await expect(service.sweepNoCredit(testJobContext())).resolves.toBe(0);
await expect(service.sweepNoCredit()).resolves.toBe(0);
expect(resumed).toEqual([]);
});
});
@@ -421,10 +298,9 @@ describe('what a plan lets an account subscribe to', () => {
await subscribe(appActor(`app-${seq}`));
expect(created[0].limits).toEqual({
perUser:
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
DEFAULT_FREE_SUBSCRIPTION
],
perUser: EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.bySubscription[
DEFAULT_FREE_SUBSCRIPTION
],
perApp: EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP.bySubscription[
DEFAULT_FREE_SUBSCRIPTION
],
@@ -28,7 +28,6 @@ import {
lookupFsSubject,
lookupKvSubject,
lookupPublicSubject,
projectScheduleTick,
type FsDeliveryContext,
type FsPublicSubject,
type KvDeliveryContext,
@@ -325,52 +324,3 @@ describe('registry typing', () => {
expect(subjects.length).toBeGreaterThan(0);
});
});
describe('the schedule tick', () => {
const tick = {
scheduleId: '2f7c1b6e-4d1a-4c55-9a8e-0e6f3c2b1a90',
scheduledFor: 1_798_761_600_000,
missed: 0,
missedCapped: false,
attempt: 1,
};
it('carries exactly the fields a handler is promised', () => {
const event = projectScheduleTick(tick);
expect(Object.keys(event).sort()).toEqual([
'attempt',
'id',
'missed',
'op',
'scheduleId',
'scheduledFor',
'subject',
]);
expect(event).toEqual({
subject: `schedule:${tick.scheduleId}`,
op: 'tick',
id: `${tick.scheduleId}:${tick.scheduledFor}`,
scheduleId: tick.scheduleId,
scheduledFor: tick.scheduledFor,
missed: 0,
attempt: 1,
});
});
it('says when `missed` stopped counting, and only then', () => {
const capped = projectScheduleTick({
...tick,
missed: 1000,
missedCapped: true,
});
expect(capped.missedCapped).toBe(true);
expect(capped.missed).toBe(1000);
expect('missedCapped' in projectScheduleTick(tick)).toBe(false);
});
it('keeps the id across attempts at one slot', () => {
expect(projectScheduleTick({ ...tick, attempt: 3 }).id).toBe(
projectScheduleTick(tick).id,
);
});
});
-45
View File
@@ -20,7 +20,6 @@
import type { EventKey, KvOp } from '../../clients/event/types.js';
import type { FSEntry } from '../../stores/fs/FSEntry.js';
import type { AclMode } from '../acl/ACLService.js';
import { scheduleToken } from '../../stores/events/types.js';
import type { NotificationAudience } from '../notification/notificationTypes.js';
import { relativeTo } from './matcher.js';
import {
@@ -360,50 +359,6 @@ const notifProject = (delivery: NotifDeliveryContext): ProjectedNotifEvent => ({
seq: delivery.seq,
});
// -- Schedule ticks ---------------------------------------------------
/**
* What a schedule's handler is handed as `event`. No bus event stands behind
* it, so it has no registry entry: the runner builds it from the slot it fires.
* Additive only, as for the other projections.
*/
export interface ScheduleEvent {
/** `schedule:<scheduleId>`. */
subject: string;
op: 'tick';
/** `${scheduleId}:${scheduledFor}`: the same across retries of one slot. */
id: string;
scheduleId: string;
/** The slot this run is for, in epoch ms. */
scheduledFor: number;
/** When the invocation was sent; stamped as it goes out. */
firedAt?: number;
/** Slots skipped since the last one that ran. */
missed: number;
/** Set when `missed` stopped counting at its cap. */
missedCapped?: true;
/** Attempts at this slot, this one included. */
attempt: number;
}
/** The {@link ScheduleEvent} of one run. Called by extensions. */
export const projectScheduleTick = (tick: {
scheduleId: string;
scheduledFor: number;
missed: number;
missedCapped: boolean;
attempt: number;
}): ScheduleEvent => ({
subject: scheduleToken(tick.scheduleId),
op: 'tick',
id: `${tick.scheduleId}:${tick.scheduledFor}`,
scheduleId: tick.scheduleId,
scheduledFor: tick.scheduledFor,
missed: tick.missed,
...(tick.missedCapped ? { missedCapped: true as const } : {}),
attempt: tick.attempt,
});
// -- Registry ---------------------------------------------------------
export const PUBLIC_SUBJECTS = [
@@ -1,124 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { HttpError } from '../../core/http/HttpError.js';
import { checkRateLimit } from '../../core/http/middleware/rateLimit.js';
import type { RouteRateLimit } from '../../core/http/types.js';
import {
HANDLER_NAME_MAX_LENGTH,
hashContent,
} from '../../stores/events/EventHandlerStore.js';
import type { HandlerScopeKind } from '../../stores/events/types.js';
/** Request and response pieces the subscription and schedule verbs share. */
export const badRequest = (message: string, code: string): HttpError =>
new HttpError(400, message, { legacyCode: code });
export const parseHandlerName = (value: unknown): string | null => {
if (value === undefined || value === null) return null;
if (typeof value !== 'string' || value.length === 0)
throw badRequest('handlerName must be a string', 'bad_request');
if (value.length > HANDLER_NAME_MAX_LENGTH)
throw badRequest(
`handlerName may not exceed ${HANDLER_NAME_MAX_LENGTH} characters`,
'bad_request',
);
return value;
};
/** Hex digest of the inline source a subscribe claims it is binding. */
export const parseHandlerHash = (value: unknown): string | null => {
if (value === undefined || value === null) return null;
if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value))
throw badRequest(
'handlerHash must be a sha-256 hex digest',
'bad_request',
);
return value;
};
/** Source of an inline handler; the store enforces the size and content rules. */
export const parseHandlerSource = (value: unknown): string | null => {
if (value === undefined || value === null) return null;
if (typeof value !== 'string' || value.trim().length === 0)
throw badRequest(
'handlerSource must be a non-empty string',
'bad_request',
);
return value;
};
/** Which handler set a request is about; absent means the app's. */
export const parseHandlerScopeKind = (
value: unknown,
field: string,
): HandlerScopeKind => {
if (value === undefined || value === null || value === 'app') return 'app';
if (value === 'user') return 'user';
throw badRequest(`${field} must be 'app' or 'user'`, 'bad_request');
};
/** Stored as JSON text; the byte cap is the store's to enforce. */
export const parseContext = (value: unknown): string | null => {
if (value === undefined || value === null) return null;
try {
return JSON.stringify(value);
} catch {
throw badRequest('context must be JSON-serializable', 'bad_request');
}
};
const tooManyCalls = (): HttpError =>
new HttpError(429, 'Too many subscription changes', {
legacyCode: 'too_many_requests',
});
/** Spend one call from a per-user window, refusing past it. */
export const spendUserWindow = async (
limit: RouteRateLimit,
userId: number,
): Promise<void> => {
const ok = await checkRateLimit(
`${limit.scope}:${userId}`,
limit.limit,
limit.window,
);
if (!ok) throw tooManyCalls();
};
/**
* A stored context as a listing may describe it: which keys it sets, and a hash
* of the whole blob. Never the values: the column is where an API key lives.
*/
export const projectContext = (
context: string | null,
): { contextKeys: string[] | null; contextHash: string | null } => {
if (context === null) return { contextKeys: null, contextHash: null };
let keys: string[] = [];
try {
const parsed: unknown = JSON.parse(context);
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
keys = Object.keys(parsed).sort();
} catch {
// Stored as JSON by the verbs, so this cannot normally happen; an
// unreadable blob still reports its hash rather than failing the list.
}
return { contextKeys: keys, contextHash: hashContent(context) };
};
@@ -27,15 +27,7 @@
* out of service so a revoked subscription stops costing anything at all.
*/
import {
afterAll,
beforeAll,
describe,
expect,
it,
onTestFinished,
vi,
} from 'vitest';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { v4 as uuidv4 } from 'uuid';
import { EVENTS_COALESCE_WINDOW_MS } from '../../controllers/events/limits.js';
import { SUSPENDED_ROW_TTL_DAYS } from '../../controllers/events/limits.js';
@@ -51,7 +43,6 @@ import type { AclMode } from '../acl/ACLService.js';
import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
import type { DeliveryEnvelope } from './EventsService.js';
import { fsAnchorToken } from './subjects.js';
import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
const BOOT_TIMEOUT_MS = 120_000;
const TABLE = 'event_subscriptions';
@@ -215,7 +206,7 @@ const makeApp = async (path: string): Promise<Actor> => {
EVENTS_BACKGROUND_PERMISSION,
);
await env.server.stores.eventHandler.publish({
scope: appHandlerScope(uid),
appUid: uid,
name: 'onChange',
source: 'async () => {}',
});
@@ -959,165 +950,3 @@ describe('what a revoked grant settles', () => {
}
});
});
describe('a holder`s user scope when consent is withdrawn', () => {
const scopeOf = (appUid: string, holder = owner.id) => ({
appUid,
holder,
});
const putUserHandler = async (appUid: string, holder = owner.id) => {
await env.server.stores.eventHandler.publish({
scope: scopeOf(appUid, holder),
name: 'mine',
source: 'async () => {}',
broadcast: false,
});
};
const held = (appUid: string, holder = owner.id) =>
env.server.stores.eventHandler.listForApp(scopeOf(appUid, holder));
let destroys: Array<{ appUid: string; scope: string; userId: number }>;
beforeAll(() => {
env.server.clients.event.on('events.worker.destroy', (_key, event) => {
destroys?.push({
appUid: event.appUid,
scope: event.scope,
userId: event.actor.user.id,
});
});
});
it('drops the scope when background consent is revoked, announcing it as the holder', async () => {
await clearRows();
destroys = [];
const path = await folder(`/${owner.username}/scope-consent`);
const app = (await makeApp(path)).app!;
await putUserHandler(app.uid);
await env.server.services.permission.revokeUserAppPermission(
owner.actor,
app.uid,
EVENTS_BACKGROUND_PERMISSION,
);
await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
timeout: 5_000,
interval: 25,
});
expect(destroys.filter((event) => event.scope === 'user')).toEqual([
{ appUid: app.uid, scope: 'user', userId: owner.id },
]);
});
it('drops the scope when the whole app is revoked', async () => {
await clearRows();
const path = await folder(`/${owner.username}/scope-whole-app`);
const app = (await makeApp(path)).app!;
await putUserHandler(app.uid);
await env.server.services.permission.revokeUserAppAll(
owner.actor,
app.uid,
);
await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
timeout: 5_000,
interval: 25,
});
});
it('leaves the scope when the consent still stands, as after a revoke then a grant', async () => {
await clearRows();
const path = await folder(`/${owner.username}/scope-regrant`);
const app = (await makeApp(path)).app!;
await putUserHandler(app.uid);
for (const permission of [EVENTS_BACKGROUND_PERMISSION, null])
await events().settleRevokedGrant({
holderUserId: owner.id,
appUid: app.uid,
permission,
});
expect(await held(app.uid)).toHaveLength(1);
});
it('leaves the scope when the consent check itself fails', async () => {
await clearRows();
const path = await folder(`/${owner.username}/scope-check-fails`);
const app = (await makeApp(path)).app!;
await putUserHandler(app.uid);
const check = vi
.spyOn(events(), 'hasBackgroundConsent')
.mockResolvedValue(null);
try {
await events().settleRevokedGrant({
holderUserId: owner.id,
appUid: app.uid,
permission: EVENTS_BACKGROUND_PERMISSION,
});
expect(check).toHaveBeenCalled();
expect(await held(app.uid)).toHaveLength(1);
} finally {
check.mockRestore();
}
});
it('leaves the scope for an unrelated grant revoked, and another holder`s scope for any', async () => {
await clearRows();
const path = await folder(`/${owner.username}/scope-unrelated`);
const app = (await makeApp(path)).app!;
await putUserHandler(app.uid);
await putUserHandler(app.uid, guest.id);
await env.server.services.permission.revokeUserAppPermission(
owner.actor,
app.uid,
`fs:${await uidOf(path)}:list`,
);
await env.server.services.permission.revokeUserAppPermission(
owner.actor,
app.uid,
EVENTS_BACKGROUND_PERMISSION,
);
await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
timeout: 5_000,
interval: 25,
});
expect(await held(app.uid, guest.id)).toHaveLength(1);
});
it('keeps the rows bound to a dropped scope, suspended rather than deleted', async () => {
await clearRows();
const path = await folder(`/${owner.username}/scope-rows`);
const appActor = await makeApp(path);
const app = appActor.app!;
const config = (events() as unknown as { config: IConfig }).config;
config.events!.userHandlers = true;
onTestFinished(() => {
config.events!.userHandlers = false;
});
const sub = (
await events().subscribeDurable(appActor, {
subject: `fs:${path}`,
handlerSource: 'async () => {}',
})
).sub;
await env.server.services.permission.revokeUserAppPermission(
owner.actor,
app.uid,
EVENTS_BACKGROUND_PERMISSION,
);
await suspendedRow(sub.subId);
await vi.waitFor(async () => expect(await held(app.uid)).toEqual([]), {
timeout: 5_000,
interval: 25,
});
expect((await rowOf(sub.subId)).suspended_reason).toBe(
'permission_revoked',
);
});
});
@@ -1,344 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The subscription lifecycle hands schedules' rows, and only theirs, to
* `services.schedules` with the change each path made. Without it every path
* still runs.
*/
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import type {
ScheduleServiceContract,
ScheduleSubscriptionChange,
} from '../../core/schedules.js';
import type { PuterServer } from '../../server.js';
import type { DurableSubscriptionInput } from '../../stores/events/DurableSubscriptionStore.js';
import {
isScheduleRow,
scheduleToken,
type DurableSubscription,
} from '../../stores/events/types.js';
import { setupTestServer } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
const BOOT_TIMEOUT_MS = 120_000;
const APP_UID = 'app-seam';
let server: PuterServer;
let userId: number;
let anchorUid: string;
let anchorPath: string;
let saved: unknown;
const durable = () => server.stores.durableSubscription;
const slot = () => server.services as unknown as { schedules?: unknown };
/** Put `schedules` in the slot, or leave the slot empty. */
const setSchedules = (schedules: unknown): void => {
if (schedules === undefined) delete slot().schedules;
else slot().schedules = schedules;
};
const recording = () => {
const calls: Array<{
subIds: string[];
change: ScheduleSubscriptionChange;
}> = [];
const schedules = {
onSubscriptionsChanged: vi.fn(
async (
rows: readonly DurableSubscription[],
change: ScheduleSubscriptionChange,
) => {
calls.push({ subIds: rows.map((r) => r.subId), change });
},
),
createFor: vi.fn(),
list: vi.fn(),
get: vi.fn(),
remove: vi.fn(),
receivePeerChanges: vi.fn(),
prepareSendAt: vi.fn(),
createDeferredAppMessage: vi.fn(),
} satisfies ScheduleServiceContract;
setSchedules(schedules);
return calls;
};
const plainRow = async (
over: Partial<DurableSubscriptionInput> = {},
): Promise<DurableSubscription> =>
(
await durable().create({
holderUserId: userId,
ownerUserId: userId,
appUid: APP_UID,
subject: `fs:${anchorPath}`,
token: `f#${anchorUid}`,
anchorUid,
anchorPath,
match: null,
op: null,
delivery: 'broadcast',
handlerName: 'onWrite',
context: null,
permission: 'list',
expiresAt: null,
...over,
})
).row;
const scheduleRow = async (
over: Partial<DurableSubscriptionInput> = {},
): Promise<DurableSubscription> => {
const uid = uuidv4();
return plainRow({
subject: scheduleToken(uid),
token: scheduleToken(uid),
anchorUid: uid,
anchorPath: '',
delivery: 'single',
handlerName: 'tick',
permission: 'events:background',
...over,
});
};
const pendingCalls = () => {
const pending = server.stores.pendingDelivery;
return [
vi.spyOn(pending, 'purge'),
vi.spyOn(pending, 'hold'),
vi.spyOn(pending, 'releaseHold'),
];
};
beforeAll(async () => {
server = await setupTestServer({ events: { enabled: true } } as IConfig);
const username = `seam-${Math.random().toString(36).slice(2, 10)}`;
const user = await server.stores.user.create({
username,
uuid: uuidv4(),
password: null,
email: `${username}@test.local`,
});
userId = user.id;
anchorPath = `/${username}/seam`;
await server.services.fs.mkdir(userId, {
path: anchorPath,
createMissingParents: true,
});
anchorUid = (await server.stores.fsEntry.getEntryByPath(anchorPath))!.uid;
saved = slot().schedules;
}, BOOT_TIMEOUT_MS);
beforeEach(async () => {
await server.clients.db.write('DELETE FROM `event_subscriptions`', []);
});
afterEach(() => {
setSchedules(saved);
vi.restoreAllMocks();
});
afterAll(async () => {
await server?.shutdown();
});
describe('the schedules lifecycle hook', () => {
it('gets only schedule rows on a suspend, and their backlog is never touched', async () => {
const calls = recording();
const schedule = await scheduleRow();
const plain = await plainRow();
const [purge, hold, releaseHold] = pendingCalls();
await server.services.events.suspendSubscriptions(
[schedule],
'no_credit',
);
expect(calls).toEqual([
{
subIds: [schedule.subId],
change: { type: 'suspended', reason: 'no_credit' },
},
]);
expect(purge).not.toHaveBeenCalled();
expect(hold).not.toHaveBeenCalled();
calls.length = 0;
await server.services.events.suspendSubscriptions([plain], 'no_credit');
expect(calls).toEqual([]);
calls.length = 0;
const suspended = (await durable().getBySubId(schedule.subId))!;
await server.services.events.resumeSubscriptions([suspended]);
expect(calls).toEqual([
{ subIds: [schedule.subId], change: { type: 'resumed' } },
]);
expect(releaseHold).not.toHaveBeenCalled();
});
it('ends schedules uninstalled when every grant to the app goes', async () => {
const calls = recording();
const schedule = await scheduleRow();
await plainRow();
await server.services.events.settleRevokedGrant({
holderUserId: userId,
appUid: APP_UID,
permission: null,
});
expect(calls).toEqual([
{
subIds: [schedule.subId],
change: {
type: 'suspended',
reason: 'permission_revoked',
endReason: 'uninstalled',
},
},
]);
});
it('reads no schedule rows to settle a specific grant, and still settles them when consent goes', async () => {
const calls = recording();
const schedule = await scheduleRow();
const plain = await plainRow();
const page = vi.spyOn(durable(), 'listActiveForHolderPage');
const readSubIds = async () =>
(await Promise.all(page.mock.results.map((r) => r.value))).flatMap(
(p) => p.rows.map((row: DurableSubscription) => row.subId),
);
await server.services.events.settleRevokedGrant({
holderUserId: userId,
appUid: APP_UID,
permission: `fs:${anchorUid}:list`,
});
const read = await readSubIds();
expect(read).toContain(plain.subId);
expect(read).not.toContain(schedule.subId);
expect(calls).toEqual([]);
page.mockClear();
await server.services.events.settleRevokedGrant({
holderUserId: userId,
appUid: APP_UID,
permission: EVENTS_BACKGROUND_PERMISSION,
});
expect(await readSubIds()).toContain(schedule.subId);
expect(calls).toEqual([
{
subIds: [schedule.subId],
change: { type: 'suspended', reason: 'permission_revoked' },
},
]);
});
it('tells the holder once per kind when an uninstall settles several pages', async () => {
recording();
const plains = [await plainRow(), await plainRow(), await plainRow()];
const schedules = [await scheduleRow(), await scheduleRow()];
const store = durable();
const listPage = store.listActiveForHolderPage.bind(store);
vi.spyOn(store, 'listActiveForHolderPage').mockImplementation(
(holder, app, afterId, _batchSize, opts) =>
listPage(holder, app, afterId, 1, opts),
);
const notify = vi.spyOn(server.services.notification, 'notify');
await expect(
server.services.events.settleRevokedGrant({
holderUserId: userId,
appUid: APP_UID,
permission: null,
}),
).resolves.toBe(5);
const ended = notify.mock.calls
.filter(([, , meta]) => meta?.type === 'app.events.ended')
.map(([, body]) => body as { count: number; subjects: string[] });
expect(ended).toHaveLength(2);
expect(ended.map((body) => body.count).sort()).toEqual([2, 3]);
expect(ended.flatMap((body) => body.subjects).sort()).toEqual(
[...plains, ...schedules].map((row) => row.subject).sort(),
);
});
it('ends schedules app_deleted when their app is deleted', async () => {
const calls = recording();
const schedule = await scheduleRow();
await plainRow();
await server.services.events.settleDeletedApp(APP_UID);
expect(calls).toEqual([
{
subIds: [schedule.subId],
change: { type: 'ended', endReason: 'app_deleted' },
},
]);
});
it('ends schedules whose rows expire', async () => {
const calls = recording();
const schedule = await scheduleRow({ expiresAt: 1 });
await plainRow({ expiresAt: 1 });
await server.services.events.sweepExpired();
const handed = calls.flatMap((c) => c.subIds);
expect(handed).toEqual([schedule.subId]);
expect(calls[0].change).toEqual({ type: 'ended', endReason: 'ended' });
});
it('lets every path run without services.schedules', async () => {
setSchedules(undefined);
const schedule = await scheduleRow();
await plainRow();
await expect(
server.services.events.suspendSubscriptions(
[schedule],
'no_credit',
),
).resolves.toBe(1);
await expect(
server.services.events.resumeSubscriptions([
(await durable().getBySubId(schedule.subId))!,
]),
).resolves.toBe(1);
await server.services.events.settleDeletedApp(APP_UID);
const left = await server.clients.db.read(
'SELECT `token` FROM `event_subscriptions`',
[],
);
expect(
left.filter((r) => isScheduleRow(r as { token: string })),
).toEqual([]);
});
});
File diff suppressed because it is too large. Load diff
@@ -1,386 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* A user-scope handler through the real local worker runtime: created inline by
* an app session, deployed eagerly as its holder, fired by an event, and run
* with the holder's identity. Also the rehydrate route's scope resolution.
*/
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import {
appHandlerScope,
hashContent,
} from '../../stores/events/EventHandlerStore.js';
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import { EVENTS_BACKGROUND_PERMISSION } from './authorization.js';
import { userHandlerScript } from './workerRuntime.js';
const BOOT_TIMEOUT_MS = 180_000;
const TEST_TIMEOUT_MS = 60_000;
const INTERNAL_SECRET = 'events-internal-secret';
const SOURCE = `async ({ event, ctx, user, fetch, ack }) => {
await fetch(ctx.sink, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
label: ctx.label,
path: event.path,
token: user.authToken,
}),
});
await ack();
}`;
interface SinkPost {
label?: string;
path?: string;
token?: string;
}
let env: PuterTestEnv;
let holders: Array<{ id: number; uuid: string; anchor: string; token: string }>;
let appUid: string;
let sink: http.Server;
let sinkUrl: string;
let sinkPosts: SinkPost[];
const localWorkers = () => env.server.services.localworkerservice;
const scriptFor = (source: string): string =>
userHandlerScript(
{ name: `@${hashContent(source)}`, sourceHash: hashContent(source) },
'',
);
const isResident = async (script: string): Promise<boolean> =>
(await localWorkers().dispatchEventsWorker(
new Request(`http://${script}.workers.puter.localhost/`, {
method: 'GET',
}),
)) !== null;
const api = async (
method: 'GET' | 'POST',
path: string,
token: string,
body?: object,
): Promise<{ status: number; body: Record<string, unknown> }> => {
const response = await fetch(new URL(path, env.apiOrigin), {
method,
headers: {
'content-type': 'application/json',
authorization: `Bearer ${token}`,
},
...(body ? { body: JSON.stringify(body) } : {}),
});
return {
status: response.status,
body: (await response.json()) as Record<string, unknown>,
};
};
const rehydrate = async (
body: object,
): Promise<{ status: number; body: Record<string, unknown> }> => {
const response = await fetch(
new URL('/events/worker/rehydrate', env.apiOrigin),
{
method: 'POST',
headers: {
'content-type': 'application/json',
'x-puter-internal-auth': INTERNAL_SECRET,
},
body: JSON.stringify(body),
},
);
return {
status: response.status,
body: (await response.json()) as Record<string, unknown>,
};
};
const createInline = async (
holder: (typeof holders)[number],
label: string,
source = SOURCE,
): Promise<string> => {
const response = await api('POST', '/events/subscribe', holder.token, {
subject: `fs:${holder.anchor}`,
delivery: 'single',
handlerSource: source,
context: { sink: sinkUrl, label },
});
expect(response.status).toBe(200);
return String(response.body.subId);
};
const delivered = (count = 1): Promise<void> =>
vi.waitFor(() => expect(sinkPosts.length).toBeGreaterThanOrEqual(count), {
timeout: 30_000,
interval: 50,
});
beforeAll(async () => {
env = await setupPuterTestEnv({
events: {
enabled: true,
workerRuntime: true,
userHandlers: true,
internalSecret: INTERNAL_SECRET,
},
workers: { localServer: true },
unlimitedMetering: true,
} as IConfig);
const owner = (await env.server.stores.user.getByUsername(
env.users.user.username,
))!;
appUid = `app-${uuidv4()}`;
await env.server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
[appUid, appUid, appUid, `https://${appUid}.example/`, owner.id],
);
holders = [];
for (const account of [env.users.user, env.users.other]) {
const user = (await env.server.stores.user.getByUsername(
account.username,
))!;
const anchor = `/${account.username}/uh-e2e`;
await env.server.services.fs.mkdir(user.id, {
path: anchor,
createMissingParents: true,
});
const entry = await env.server.stores.fsEntry.getEntryByPath(anchor);
const { actor } = await env.server.services.auth.authenticate(
account.token,
);
for (const permission of [
`fs:${entry!.uid}:list`,
EVENTS_BACKGROUND_PERMISSION,
])
await env.server.services.permission.grantUserAppPermission(
actor!,
appUid,
permission,
);
holders.push({
id: user.id,
uuid: user.uuid,
anchor,
token: await env.server.services.auth.getUserAppToken(
actor!,
appUid,
),
});
}
sinkPosts = [];
sink = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on('data', (chunk: Buffer) => chunks.push(chunk));
req.on('end', () => {
sinkPosts.push(
JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'),
);
res.writeHead(200).end();
});
});
await new Promise<void>((resolve) =>
sink.listen(0, '127.0.0.1', () => resolve()),
);
sinkUrl = `http://127.0.0.1:${(sink.address() as AddressInfo).port}`;
}, BOOT_TIMEOUT_MS);
afterAll(async () => {
await new Promise<void>((resolve) => {
if (!sink) return resolve();
sink.closeAllConnections();
sink.close(() => resolve());
});
await env?.shutdown();
});
beforeEach(() => {
sinkPosts = [];
});
describe('an inline handler in a user scope, end to end', () => {
it(
'is deployed eagerly, fires on an event and runs as its holder',
{ timeout: TEST_TIMEOUT_MS },
async () => {
const [holder] = holders;
const source = `${SOURCE}\n// run-as-holder`;
const script = scriptFor(source);
expect(await isResident(script)).toBe(false);
await createInline(holder, 'inline', source);
// The deploy started with the publish, before any event.
await vi.waitFor(async () =>
expect(await isResident(script)).toBe(true),
);
await env.server.services.fs.touch(holder.id, {
path: `${holder.anchor}/note.txt`,
});
await delivered();
expect(sinkPosts[0]).toMatchObject({
label: 'inline',
path: `${holder.anchor}/note.txt`,
});
const { actor } = await env.server.services.auth.authenticate(
sinkPosts[0].token!,
);
expect(actor!.user?.id).toBe(holder.id);
expect(actor!.effectiveApp?.uid).toBe(appUid);
},
);
it(
'shares one deployed script between holders of the same code, each run as its own holder',
{ timeout: TEST_TIMEOUT_MS },
async () => {
const source = `${SOURCE}\n// shared-code`;
const script = scriptFor(source);
const create = vi.spyOn(env.server.drivers.workers, 'create');
await createInline(holders[0], 'first', source);
await vi.waitFor(async () =>
expect(await isResident(script)).toBe(true),
);
await createInline(holders[1], 'second', source);
// Let the second holder's eager deploy settle before counting.
await new Promise((resolve) => setTimeout(resolve, 500));
// The marker made the second holder's eager deploy a no-op.
expect(
create.mock.calls.filter(
([args]) =>
(args as { workerName: string }).workerName === script,
),
).toHaveLength(1);
for (const [i, holder] of holders.entries())
await env.server.services.fs.touch(holder.id, {
path: `${holder.anchor}/shared-${i}.txt`,
});
await delivered(2);
const runAs = new Set<number>();
for (const post of sinkPosts)
runAs.add(
(await env.server.services.auth.authenticate(post.token!))
.actor!.user!.id,
);
expect(runAs).toEqual(new Set(holders.map((h) => h.id)));
create.mockRestore();
},
);
});
describe('the rehydrate route', () => {
const source = `${SOURCE}\n// rehydrate`;
const script = scriptFor(source);
beforeAll(async () => {
// Rows only: the handler is published, nothing is deployed here.
const eventHandler = env.server.stores.eventHandler;
await eventHandler.publish({
scope: { appUid, holder: holders[0].id },
name: `@${hashContent(source)}`,
source,
broadcast: false,
});
await eventHandler.publish({
scope: { appUid, holder: holders[1].id },
name: 'other',
source: SOURCE,
broadcast: false,
});
});
it('resolves the holder`s user scope when the body names one', async () => {
const response = await rehydrate({
script,
appUid,
holder: holders[0].uuid,
});
expect(response).toMatchObject({
status: 200,
body: { deployed: true },
});
expect(await isResident(script)).toBe(true);
});
it('reads an absent or empty holder as the app scope, which has no such handler', async () => {
for (const body of [
{ script, appUid },
{ script, appUid, holder: '' },
])
expect(await rehydrate(body)).toMatchObject({
status: 404,
body: { deployed: false, reason: 'no-handlers' },
});
expect(
await env.server.stores.eventHandler.listForApp(
appHandlerScope(appUid),
),
).toEqual([]);
});
it('answers stale for a holder whose scope does not hold that handler', async () => {
expect(
await rehydrate({ script, appUid, holder: holders[1].uuid }),
).toMatchObject({
status: 404,
body: { deployed: false, reason: 'stale' },
});
});
it('refuses a holder that does not exist, and a malformed one', async () => {
expect(
await rehydrate({ script, appUid, holder: uuidv4() }),
).toMatchObject({
status: 404,
body: { deployed: false, reason: 'no-holder' },
});
expect(
(await rehydrate({ script, appUid, holder: 'a b' })).status,
).toBe(400);
expect((await rehydrate({ script, appUid, holder: 7 })).status).toBe(
400,
);
});
});
@@ -34,7 +34,6 @@ import type { AddressInfo } from 'node:net';
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
@@ -59,14 +58,8 @@ import {
EVENTS_WORKER_SESSION_NAME,
eventsInvokeKey,
eventsWorkerScript,
userHandlerScript,
} from './workerRuntime.js';
import { handlerSetHash } from './workerSource.js';
import {
appHandlerScope,
hashContent,
} from '../../stores/events/EventHandlerStore.js';
import type { WorkerInvocation } from './workerSeam.js';
const BOOT_TIMEOUT_MS = 120_000;
/** Short enough that a hung handler is a test case rather than a stall. */
@@ -136,7 +129,6 @@ const startStub = async (): Promise<string> => {
script: req.headers['x-puter-events-script'] as string,
app: req.headers['x-puter-events-app'] as string,
key: req.headers['x-puter-events-key'] as string,
holder: req.headers['x-puter-events-holder'] as string,
deployed,
},
body: JSON.parse(raw || '{}') as StubCall['body'],
@@ -241,7 +233,7 @@ const makeWorkerApp = async (): Promise<{
);
const token = await env.server.services.auth.getUserAppToken(actor!, uid);
await env.server.stores.eventHandler.publish({
scope: appHandlerScope(uid),
appUid: uid,
name: HANDLER,
source: SOURCE,
});
@@ -411,7 +403,7 @@ beforeAll(async () => {
);
appToken = await env.server.services.auth.getUserAppToken(actor!, appUid);
await env.server.stores.eventHandler.publish({
scope: appHandlerScope(appUid),
appUid,
name: HANDLER,
source: SOURCE,
});
@@ -479,9 +471,7 @@ describe('the call an owed delivery makes', () => {
const script = eventsWorkerScript(
handlerSetHash(
await env.server.stores.eventHandler.setForApp(
appHandlerScope(appUid),
),
await env.server.stores.eventHandler.setForApp(appUid),
),
// No `workers.internetExposedUrl` in this test's config.
'',
@@ -1024,10 +1014,7 @@ describe('with no events worker to address', () => {
const subId = await subscribe();
// Unpublished after the fact: the app's set now hashes to nothing, so
// there is no script to name and nowhere to send the delivery.
await env.server.stores.eventHandler.remove(
appHandlerScope(appUid),
HANDLER,
);
await env.server.stores.eventHandler.remove(appUid, HANDLER);
try {
await touch('unresolved.txt');
@@ -1050,7 +1037,7 @@ describe('with no events worker to address', () => {
expect(calls).toEqual([]);
} finally {
await env.server.stores.eventHandler.publish({
scope: appHandlerScope(appUid),
appUid,
name: HANDLER,
source: SOURCE,
});
@@ -1429,349 +1416,3 @@ describe('handlers whose writes run handlers', () => {
).resolves.toBeNull();
});
});
describe('a handler in a user scope', () => {
const USER_SOURCE =
'async ({ event, ctx }) => { console.log("user", event.path, ctx.label); }';
const userName = `@${hashContent(USER_SOURCE)}`;
let holderUuid: string;
let createSpy: ReturnType<typeof vi.spyOn>;
let meterSpy: ReturnType<typeof vi.spyOn>;
const setFlag = (on: boolean): void => {
(
events() as unknown as { config: IConfig }
).config.events!.userHandlers = on;
};
const userScope = () => ({ appUid, holder: userId });
const userScript = () =>
userHandlerScript(
{ name: userName, sourceHash: hashContent(USER_SOURCE) },
'',
);
const post = async (path: string, body: object) => {
const response = await fetch(new URL(path, env.apiOrigin), {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${appToken}`,
},
body: JSON.stringify(body),
});
return {
status: response.status,
body: (await response.json()) as Record<string, unknown>,
};
};
/** A single-delivery row bound to a handler published by its own source. */
const inlineRow = async (source = USER_SOURCE): Promise<string> => {
const created = await post('/events/subscribe', {
subject: `fs:${anchor}`,
delivery: 'single',
handlerSource: source,
context: { label: 'user' },
});
expect(created.status).toBe(200);
return String(created.body.subId);
};
const invocationFor = (subId: string): WorkerInvocation => ({
subId,
holderUserId: userId,
appUid,
handlerName: userName,
handlerScope: 'user',
event: {
subject: 'schedule:s1',
op: 'tick',
id: 's1:1',
scheduleId: 's1',
scheduledFor: 1,
missed: 0,
attempt: 1,
},
context: null,
handlerDepth: 0,
});
beforeAll(async () => {
holderUuid = (await env.server.stores.user.getById(userId))!.uuid;
});
beforeEach(async () => {
setFlag(true);
await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
// Eager and on-miss deploys have no deploy target in this suite.
createSpy = vi
.spyOn(env.server.drivers.workers, 'create')
.mockResolvedValue({ success: true, errors: [] });
meterSpy = vi.spyOn(
env.server.services.metering,
'bufferIncrementUsages',
);
});
afterEach(async () => {
createSpy.mockRestore();
meterSpy.mockRestore();
setFlag(false);
await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
});
it('addresses the handler`s own script and names its holder', async () => {
const subId = await inlineRow();
await touch('user-scope.txt');
await invoked(1);
const call = calls[0];
expect(call.headers.script).toBe(userScript());
expect(call.headers.holder).toBe(holderUuid);
expect(call.headers.key).toBe(
eventsInvokeKey(INTERNAL_SECRET, userScript()),
);
expect(call.body.handler).toBe(userName);
await waitUntil(async () =>
expect(await pending().depth(subId)).toBe(0),
);
});
it('sends no holder for an app-scope row', async () => {
await subscribe();
await touch('app-scope.txt');
await invoked(1);
expect(calls[0].headers.holder).toBeUndefined();
});
it('keeps a user row and an app row of the same name on their own scope`s handler', async () => {
await env.server.stores.eventHandler.publish({
scope: userScope(),
name: HANDLER,
source: USER_SOURCE,
});
const created = await post('/events/subscribe', {
subject: `fs:${anchor}`,
delivery: 'single',
handlerName: HANDLER,
handlerScope: 'user',
});
expect(created.status).toBe(200);
await touch('same-name-user.txt');
await invoked(1);
// One at a time: a second row firing alongside would go as a batch.
expect(
(
await post('/events/unsubscribe', {
subId: String(created.body.subId),
})
).status,
).toBe(200);
await subscribe();
await touch('same-name-app.txt');
await invoked(2);
const appScript = eventsWorkerScript(
handlerSetHash(
await env.server.stores.eventHandler.setForApp(
appHandlerScope(appUid),
),
),
'',
);
const mine = userHandlerScript(
{ name: HANDLER, sourceHash: hashContent(USER_SOURCE) },
'',
);
expect(mine).not.toBe(appScript);
expect(calls[0].headers).toMatchObject({
script: mine,
holder: holderUuid,
});
expect(calls[1].headers.script).toBe(appScript);
expect(calls[1].headers.holder).toBeUndefined();
});
it('suspends a row whose handler left its holder`s set, and tells the holder only', async () => {
const notify = vi.spyOn(env.server.services.notification, 'notify');
const subId = await inlineRow();
// Gone without the removal path that would suspend its rows.
await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
await touch('missing.txt');
await waitUntil(async () =>
expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
'handler_not_found',
),
);
expect(calls).toEqual([]);
expect(notify).toHaveBeenCalledWith(
[userId],
expect.objectContaining({ handler: userName }),
expect.objectContaining({
type: 'app.events.handler_suspended',
appUid,
}),
);
expect(notify).not.toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
expect.objectContaining({ type: 'app.events.suspended' }),
);
notify.mockRestore();
});
it('does not suspend a schedule run whose replica read lags the publish', async () => {
const subId = await inlineRow();
const store = env.server.stores.eventHandler;
const real = store.setForApp.bind(store);
// The replica has not seen the insert; the primary has.
const lagging = vi
.spyOn(store, 'setForApp')
.mockImplementation(async (scope, options) =>
options?.replica
? [{ name: 'other', sourceHash: 'a'.repeat(64) }]
: real(scope, options),
);
try {
const result = await events().invokeForSchedule(
invocationFor(subId),
);
expect(result).toMatchObject({ outcome: 'settled' });
expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
expect(calls[0].headers.script).toBe(userScript());
} finally {
lagging.mockRestore();
}
});
it('reads a cold user scope from the replica only, once, for a delivery', async () => {
await inlineRow();
// Eager deploy at publish has finished with the primary by now.
await new Promise((resolve) => setTimeout(resolve, 300));
const redis = env.server.clients.redis;
const key = `eventHandlers:uset:${appUid}:${userId}`;
await redis.del(key, `${key}:r`);
const db = env.server.clients.db;
const primary = vi.spyOn(db, 'pread');
const replica = vi.spyOn(db, 'read');
const handlerReads = (spy: typeof primary) =>
spy.mock.calls.filter(([sql]) =>
String(sql).includes('event_user_handlers'),
).length;
await touch('cold-user-scope.txt');
await invoked(1);
expect(handlerReads(primary)).toBe(0);
expect(handlerReads(replica)).toBe(1);
primary.mockRestore();
replica.mockRestore();
});
it('resumes a suspended row when the name is published again', async () => {
const subId = await inlineRow();
await env.server.stores.eventHandler.deleteUserScope(appUid, userId);
await touch('gone.txt');
await waitUntil(async () =>
expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
'handler_not_found',
),
);
expect(
(
await post('/events/handlers/publish', {
scope: 'user',
name: userName,
source: USER_SOURCE,
})
).status,
).toBe(200);
expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
});
it('does not address a holder that is suspended, and leaves the row owed', async () => {
const subId = await inlineRow();
await env.server.clients.db.write(
'UPDATE user SET suspended = 1 WHERE id = ?',
[userId],
);
await env.server.stores.user.invalidateById(userId);
try {
await touch('suspended-holder.txt');
await failuresRecorded(subId, 1);
expect(calls).toEqual([]);
expect(await pending().depth(subId)).toBe(1);
expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
} finally {
await env.server.clients.db.write(
'UPDATE user SET suspended = 0 WHERE id = ?',
[userId],
);
await env.server.stores.user.invalidateById(userId);
}
});
it('deploys the holder`s script on a dispatcher miss, as that holder', async () => {
dispatchMissing = true;
await inlineRow();
await touch('miss.txt');
await invoked(2);
expect(calls[0].headers.deployed).toBeUndefined();
expect(calls[1].headers.deployed).toBe('1');
// Once for the eager deploy at publish, once for the miss.
const deployed = createSpy.mock.calls.filter(
([args]) =>
(args as { workerName: string }).workerName === userScript(),
);
expect(deployed.length).toBeGreaterThanOrEqual(1);
});
describe('with the flag off', () => {
it('leaves the delivery owed: nothing is called, suspended or billed', async () => {
const subId = await inlineRow();
setFlag(false);
meterSpy.mockClear();
await touch('flag-off.txt');
await new Promise((resolve) => setTimeout(resolve, 400));
expect(calls).toEqual([]);
expect(await pending().depth(subId)).toBe(1);
expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
expect(meterSpy).not.toHaveBeenCalled();
});
it('answers unavailable from the invoker and for a schedule run', async () => {
const subId = await inlineRow();
setFlag(false);
const invocation = invocationFor(subId);
const worker = events().worker as unknown as {
run: (i: WorkerInvocation) => Promise<unknown>;
};
expect(await worker.run(invocation)).toEqual({
outcome: 'unavailable',
});
expect(await events().invokeForSchedule(invocation)).toEqual({
outcome: 'unavailable',
});
expect(calls).toEqual([]);
expect((await durable().getBySubId(subId))?.suspendedAt).toBeNull();
});
it('leaves an app-scope row alone', async () => {
setFlag(false);
await subscribe();
await touch('app-row-flag-off.txt');
await invoked(1);
expect(calls[0].headers.script).toMatch(/^evw-/);
});
});
it('runs a schedule invocation of a user scope with its holder', async () => {
const subId = await inlineRow();
const result = await events().invokeForSchedule(invocationFor(subId));
expect(result).toMatchObject({ outcome: 'settled' });
expect(calls[0].headers.script).toBe(userScript());
expect(calls[0].headers.holder).toBe(holderUuid);
});
});
@@ -61,7 +61,6 @@ import {
eventsWorkerScript,
} from './workerRuntime.js';
import { handlerSetHash } from './workerSource.js';
import { appHandlerScope } from '../../stores/events/EventHandlerStore.js';
const BOOT_TIMEOUT_MS = 180_000;
const TEST_TIMEOUT_MS = 60_000;
@@ -172,7 +171,7 @@ const localWorkers = () => env.server.services.localworkerservice;
/** The script the app's published set currently deploys as. */
const scriptName = async (): Promise<string> =>
eventsWorkerScript(
handlerSetHash(await env.server.stores.eventHandler.setForApp(appHandlerScope(appUid))),
handlerSetHash(await env.server.stores.eventHandler.setForApp(appUid)),
// No `workers.internetExposedUrl` in this test's config, so the scope
// this backend binds is empty.
'',
@@ -887,7 +886,7 @@ describe('the worker follows the handler set', () => {
// An empty set names no script, so nothing can be deployed for
// this app again until something is published.
expect(
await env.server.stores.eventHandler.setForApp(appHandlerScope(appUid)),
await env.server.stores.eventHandler.setForApp(appUid),
).toEqual([]);
expect((await durable().getBySubId(subId))?.suspendedReason).toBe(
'handler_not_found',
@@ -23,9 +23,7 @@ import {
eventsInvokeKey,
eventsWorkerScope,
eventsWorkerScript,
userHandlerScript,
} from './workerRuntime.js';
import { handlerSetHash } from './workerSource.js';
const SET_HASH = 'a1'.repeat(32);
const OTHER_HASH = 'b2'.repeat(32);
@@ -52,37 +50,6 @@ describe('eventsWorkerScript', () => {
});
});
describe('userHandlerScript', () => {
const handler = { name: '@abc', sourceHash: SET_HASH };
it('is the script of the handler as a set of one', () => {
expect(userHandlerScript(handler, '')).toBe(
eventsWorkerScript(handlerSetHash([handler]), ''),
);
});
it('is the same for identical code whoever holds it, and differs by code and name', () => {
// Nothing but name and source hash goes in: two holders (or two apps)
// of the same handler deploy one script.
const again = { ...handler };
expect(userHandlerScript(again, 's')).toBe(
userHandlerScript(handler, 's'),
);
expect(
userHandlerScript({ ...handler, sourceHash: OTHER_HASH }, 's'),
).not.toBe(userHandlerScript(handler, 's'));
expect(userHandlerScript({ ...handler, name: '@abd' }, 's')).not.toBe(
userHandlerScript(handler, 's'),
);
});
it('stays scoped to the backend', () => {
expect(userHandlerScript(handler, 'https://a.example')).not.toBe(
userHandlerScript(handler, 'https://b.example'),
);
});
});
describe('eventsWorkerScope', () => {
it('reads the same field `#cfDeploy` binds as `puter_endpoint`, and falls back to empty', () => {
expect(eventsWorkerScope({})).toBe('');
@@ -18,8 +18,6 @@
*/
import { createHash, createHmac } from 'node:crypto';
import type { EventHandler } from '../../stores/events/EventHandlerStore.js';
import { handlerSetHash } from './workerSource.js';
/**
* How an app's events worker is named and how it recognizes the platform.
@@ -73,17 +71,6 @@ export const eventsWorkerScript = (setHash: string, scope: string): string =>
.digest('hex')
.slice(0, 32)}`;
/**
* The script one user-scope handler deploys as: the set-of-one name, built from
* the handler's name and source hash alone. The holder and the app are not part
* of it, so identical code under one name is one script however many users and
* apps publish it.
*/
export const userHandlerScript = (
handler: Pick<EventHandler, 'name' | 'sourceHash'>,
scope: string,
): string => eventsWorkerScript(handlerSetHash([handler]), scope);
/** Version prefix on a derived key, so a rotation is visible in a log line. */
export const EVENTS_INVOKE_KEY_VERSION = 'k1';
+19 -96
View File
@@ -17,15 +17,8 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type {
EventsWorkerInvokerClient,
WorkerInvokeOptions,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import type { Actor } from '../../core/actor.js';
import type { HandlerScopeKind } from '../../stores/events/types.js';
import type { DeliverableEvent, ScheduleEvent } from './registry.js';
export type { ScheduleEvent } from './registry.js';
import type { EventsWorkerInvokerClient } from '../../clients/events/EventsWorkerInvokerClient.js';
import type { DeliverableEvent } from './registry.js';
/**
* Where a delivery leaves the event system for the app's own code.
@@ -43,15 +36,11 @@ export interface WorkerInvocation {
/** The app whose handler runs, or null for an account-owned row. */
appUid: string | null;
handlerName: string | null;
/** Whose handler set `handlerName` is in: absent means the app's own. */
handlerScope?: HandlerScopeKind;
event: DeliverableEvent | ScheduleEvent;
event: DeliverableEvent;
/** The subscription's stored context, delivered to the handler as `ctx`. */
context: string | null;
/** Handler runs behind `event`; the token this run is handed is one deeper. */
handlerDepth: number;
/** The holder acting as the app, when the caller resolved it already. */
actor?: Actor;
}
/**
@@ -75,12 +64,6 @@ export interface WorkerInvokerSeam {
available?(): boolean;
}
/**
* Why no token could be minted: nobody to run as, no consent to, or the mint
* itself failed.
*/
export type MintRefusal = 'no-identity' | 'no-consent' | 'mint-failed';
/**
* Mints the subscriber's app-under-user token for one invocation. The
* `events:background` consent is what authorizes it, not a narrower
@@ -88,35 +71,15 @@ export type MintRefusal = 'no-identity' | 'no-consent' | 'mint-failed';
*/
export type SubscriberTokenMinter = (
invocation: WorkerInvocation,
) => Promise<{ token: string } | { refused: MintRefusal }>;
/** Where the handler set is read from when its cache misses. */
export interface AddressOptions {
missRead?: 'primary' | 'replica';
/** A handler the set must have; a set without it is re-read on the primary. */
handlerName?: string;
/** Absent means the app's own set. */
handlerScope?: HandlerScopeKind;
/** The user whose scope it is, for `handlerScope: 'user'`. */
holderUserId?: number;
}
) => Promise<string | null>;
/**
* The script a handler deploys as and its invocation key; null when it has none
* to run, no events secret is configured, or (in a user scope) the holder is
* gone or suspended. `holder` is the holder's uuid, for a user scope.
* The script an app's handlers deploy as and its invocation key; null when the
* app has no handlers or no events secret is configured.
*/
export type EventsWorkerAddresser = (
appUid: string,
options?: AddressOptions,
) => Promise<{ script: string; key: string; holder?: string } | null>;
/** What one run came to, including why nothing was invoked. */
export type WorkerRunResult =
WorkerInvocationResult | { outcome: MintRefusal | 'deploy-busy' };
/** How {@link EventsWorkerInvoker.run} reaches the worker. */
export type WorkerRunOptions = WorkerInvokeOptions & AddressOptions;
) => Promise<{ script: string; key: string } | null>;
/** How often the same script's retriable failures are logged. */
const WARN_INTERVAL_MS = 60_000;
@@ -134,7 +97,6 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
readonly #mintToken: SubscriberTokenMinter;
readonly #address: EventsWorkerAddresser;
readonly #available: () => boolean;
readonly #userHandlersEnabled: () => boolean;
/**
* Script -> when it was last logged, so a failing script floods once a
* minute.
@@ -146,13 +108,11 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
mintToken: SubscriberTokenMinter,
address: EventsWorkerAddresser,
available: () => boolean,
userHandlersEnabled: () => boolean = () => true,
) {
this.#client = client;
this.#mintToken = mintToken;
this.#address = address;
this.#available = available;
this.#userHandlersEnabled = userHandlersEnabled;
}
available(): boolean {
@@ -162,65 +122,28 @@ export class EventsWorkerInvoker implements WorkerInvokerSeam {
async invoke(
invocation: WorkerInvocation,
): Promise<WorkerInvocationResult> {
const result = await this.run(invocation);
switch (result.outcome) {
// Nothing was attempted.
case 'no-identity':
case 'no-consent':
case 'mint-failed':
return { outcome: 'deferred' };
case 'deploy-busy':
return { outcome: 'unavailable' };
default:
return result;
}
}
/** {@link invoke}, saying why nothing was invoked. */
async run(
invocation: WorkerInvocation,
options: WorkerRunOptions = {},
): Promise<WorkerRunResult> {
const { appUid, handlerName } = invocation;
if (!appUid || !handlerName) return { outcome: 'deferred' };
if (!this.#available()) return { outcome: 'unavailable' };
// Owed, not failed: nothing resolves a user scope while it is off.
const { handlerScope } = invocation;
if (handlerScope === 'user' && !this.#userHandlersEnabled())
return { outcome: 'unavailable' };
const minted = await this.#mintToken(invocation);
if ('refused' in minted) return { outcome: minted.refused };
const token = await this.#mintToken(invocation);
if (token === null) return { outcome: 'deferred' };
// Nowhere to send it: the app has no handlers or no owner to run them
// as. Retrying cannot fix that, so the consecutive-failure rule is what
// stops it.
const address = await this.#address(appUid, {
missRead: options.missRead,
handlerName,
...(handlerScope === 'user'
? { handlerScope, holderUserId: invocation.holderUserId }
: {}),
});
const address = await this.#address(appUid);
if (address === null) return { outcome: 'retriable' };
const result = await this.#client.invoke(
{
script: address.script,
key: address.key,
appUid,
...(address.holder ? { holder: address.holder } : {}),
handler: handlerName,
token: minted.token,
event: invocation.event,
ctx: parseContext(invocation.context),
},
{
lane: options.lane,
stampEvent: options.stampEvent,
},
);
if (result.deployBusy) return { outcome: 'deploy-busy' };
const result = await this.#client.invoke({
script: address.script,
key: address.key,
appUid,
handler: handlerName,
token,
event: invocation.event,
ctx: parseContext(invocation.context),
});
if (
(result.outcome === 'retriable' ||
result.outcome === 'unavailable') &&
@@ -175,7 +175,7 @@ let destroys: Array<{ appUid: string; ownerId: number | undefined }>;
beforeAll(async () => {
env = await setupPuterTestEnv({
events: { enabled: true, userHandlers: true },
events: { enabled: true },
unlimitedMetering: true,
} as IConfig);
@@ -204,7 +204,6 @@ afterAll(async () => {
beforeEach(async () => {
await env.server.clients.db.write('DELETE FROM `event_handlers`', []);
await env.server.clients.db.write('DELETE FROM `event_user_handlers`', []);
await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []);
await env.server.clients.db.write('DELETE FROM `apps`', []);
creates.length = 0;
@@ -540,8 +539,6 @@ describe('POST /events/workers/destroy', () => {
const suspended = await env.server.stores.durableSubscription.listByHandler(
app.uid,
'app',
0,
'ingestUpload',
{ suspendedReason: 'handler_not_found' },
);
@@ -557,8 +554,6 @@ describe('POST /events/workers/destroy', () => {
const stillSuspended =
await env.server.stores.durableSubscription.listByHandler(
app.uid,
'app',
0,
'ingestUpload',
{ suspendedReason: 'handler_not_found' },
);
@@ -626,319 +621,3 @@ describe('POST /events/workers/destroy', () => {
expect(destroys).toEqual([]);
});
});
// -- User scopes -----------------------------------------------------
/** `userId`'s own session of `app`, wherever the app's owner is. */
const sessionIn = async (app: { uid: string }, asUser: 'user' | 'other') => {
const { actor } = await env.server.services.auth.authenticate(
asUser === 'user' ? env.users.user.token : env.users.other.token,
);
return env.server.services.auth.getUserAppToken(actor!, app.uid);
};
const publishUserScope = (token: string, name: string, source: string) =>
call('POST', '/events/handlers/publish', token, {
scope: 'user',
name,
source,
});
describe('GET /events/workers with user scopes', () => {
it('adds the caller`s user scopes to their app workers outside an app', async () => {
const mine = await makeApp(userId);
await publish(mine.token, {
appUid: mine.uid,
name: 'a',
source: SOURCE,
});
const theirs = await makeApp(otherUserId);
const inTheirs = await sessionIn(theirs, 'user');
await publishUserScope(inTheirs, 'x', SOURCE);
await publishUserScope(inTheirs, 'y', OTHER_SOURCE);
// Another holder's scope in the same app is not the caller's.
await publishUserScope(await sessionIn(theirs, 'other'), 'z', SOURCE);
const listed = await call(
'GET',
'/events/workers',
env.users.user.token,
);
expect(listed.status).toBe(200);
expect(listed.body.items).toEqual([
expect.objectContaining({
scope: 'app',
appUid: mine.uid,
handlerCount: 1,
}),
expect.objectContaining({
scope: 'user',
appUid: theirs.uid,
appName: theirs.uid,
handlerCount: 2,
}),
]);
expect(listed.body.cursor).toBeUndefined();
});
it('pages through the app workers and then the user scopes', async () => {
const mine = await makeApp(userId);
await publish(mine.token, {
appUid: mine.uid,
name: 'a',
source: SOURCE,
});
const first = await makeApp(otherUserId);
const second = await makeApp(otherUserId);
await publishUserScope(await sessionIn(first, 'user'), 'x', SOURCE);
await publishUserScope(await sessionIn(second, 'user'), 'x', SOURCE);
const seen: Array<Record<string, unknown>> = [];
let cursor: string | undefined;
let pages = 0;
do {
const page = await call(
'GET',
`/events/workers?limit=1${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`,
env.users.user.token,
);
expect(page.status).toBe(200);
seen.push(...(page.body.items as Array<Record<string, unknown>>));
cursor = page.body.cursor as string | undefined;
pages += 1;
} while (cursor && pages < 10);
expect(seen.map((item) => [item.scope, item.appUid])).toEqual([
['app', mine.uid],
...[first.uid, second.uid].sort().map((uid) => ['user', uid]),
]);
});
it('lists only that app`s scope for the caller inside an app', async () => {
const theirs = await makeApp(otherUserId);
const otherApp = await makeApp(otherUserId);
const inTheirs = await sessionIn(theirs, 'user');
await publishUserScope(inTheirs, 'x', SOURCE);
await publishUserScope(await sessionIn(otherApp, 'user'), 'x', SOURCE);
await publishUserScope(await sessionIn(theirs, 'other'), 'x', SOURCE);
// The owner's own worker in that app is not the caller's to see.
await publish(theirs.token, {
appUid: theirs.uid,
name: 'owner',
source: SOURCE,
});
const listed = await call('GET', '/events/workers', inTheirs);
expect(listed.status).toBe(200);
expect(listed.body.items).toEqual([
expect.objectContaining({
scope: 'user',
appUid: theirs.uid,
handlerCount: 1,
}),
]);
});
it('still lists the owner`s worker beside their own user scope inside the app', async () => {
const mine = await makeApp(userId);
await publish(mine.token, {
appUid: mine.uid,
name: 'a',
source: SOURCE,
});
await publishUserScope(mine.token, 'x', SOURCE);
const listed = await call('GET', '/events/workers', mine.token);
expect(
(listed.body.items as Array<Record<string, unknown>>).map(
(item) => item.scope,
),
).toEqual(['app', 'user']);
});
it('refuses a scoped token, which has no scope of its own', async () => {
const theirs = await makeApp(otherUserId);
await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
const scoped = await appIssuedScopedToken({
uid: theirs.uid,
token: await sessionIn(theirs, 'user'),
});
const listed = await call('GET', '/events/workers', scoped);
expect(listed.status).toBe(403);
expect(listed.body.code).toBe('events_worker_owner_only');
});
});
describe('POST /events/workers/destroy with a user scope', () => {
it('removes the caller`s scope only, suspends what was bound, and announces it', async () => {
const theirs = await makeApp(otherUserId);
const mineIn = await sessionIn(theirs, 'user');
const otherIn = await sessionIn(theirs, 'other');
await publish(theirs.token, {
appUid: theirs.uid,
name: 'owner',
source: SOURCE,
});
await publishUserScope(mineIn, 'x', SOURCE);
await publishUserScope(mineIn, 'y', OTHER_SOURCE);
await publishUserScope(otherIn, 'x', SOURCE);
destroys.length = 0;
const destroyed = await call(
'POST',
'/events/workers/destroy',
mineIn,
{
scope: 'user',
},
);
expect(destroyed.status).toBe(200);
expect(destroyed.body).toMatchObject({
appUid: theirs.uid,
removed: 2,
suspended: 0,
scope: 'user',
});
const store = env.server.stores.eventHandler;
expect(
await store.countForApp({ appUid: theirs.uid, holder: userId }),
).toBe(0);
expect(
await store.countForApp({
appUid: theirs.uid,
holder: otherUserId,
}),
).toBe(1);
expect(await store.countForApp({ appUid: theirs.uid, holder: 0 })).toBe(
1,
);
expect(destroys).toEqual([{ appUid: theirs.uid, ownerId: userId }]);
const again = await call('POST', '/events/workers/destroy', mineIn, {
scope: 'user',
});
expect(again.status).toBe(404);
});
it('refuses a scoped token and an app session naming another app', async () => {
const theirs = await makeApp(otherUserId);
const mineIn = await sessionIn(theirs, 'user');
await publishUserScope(mineIn, 'x', SOURCE);
const scoped = await appIssuedScopedToken({
uid: theirs.uid,
token: mineIn,
});
const elsewhere = await makeApp(otherUserId);
for (const [token, body] of [
[scoped, { scope: 'user' }],
[mineIn, { scope: 'user', appUid: elsewhere.uid }],
] as const) {
const refused = await call(
'POST',
'/events/workers/destroy',
token,
body,
);
expect(refused.status).toBe(403);
expect(refused.body.code).toBe('events_handler_forbidden');
}
expect(
await env.server.stores.eventHandler.countForApp({
appUid: theirs.uid,
holder: userId,
}),
).toBe(1);
});
});
describe('destroying a user scope from an account session', () => {
it('removes the caller`s own scope of any app, and nobody else`s', async () => {
const theirs = await makeApp(otherUserId);
await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
await publishUserScope(await sessionIn(theirs, 'other'), 'x', SOURCE);
const store = env.server.stores.eventHandler;
destroys.length = 0;
const destroyed = await call(
'POST',
'/events/workers/destroy',
env.users.user.token,
{ scope: 'user', appUid: theirs.uid },
);
expect(destroyed.status).toBe(200);
expect(destroyed.body).toMatchObject({ removed: 1, scope: 'user' });
expect(
await store.countForApp({ appUid: theirs.uid, holder: userId }),
).toBe(0);
expect(
await store.countForApp({ appUid: theirs.uid, holder: otherUserId }),
).toBe(1);
expect(destroys).toEqual([{ appUid: theirs.uid, ownerId: userId }]);
// The same call as the other user removes only theirs; the first user
// has no scope left to touch.
const none = await call(
'POST',
'/events/workers/destroy',
env.users.user.token,
{ scope: 'user', appUid: theirs.uid },
);
expect(none.status).toBe(404);
expect(
await store.countForApp({ appUid: theirs.uid, holder: otherUserId }),
).toBe(1);
});
it('needs an appUid, and still refuses a scoped token', async () => {
const theirs = await makeApp(otherUserId);
const mineIn = await sessionIn(theirs, 'user');
await publishUserScope(mineIn, 'x', SOURCE);
const noApp = await call(
'POST',
'/events/workers/destroy',
env.users.user.token,
{ scope: 'user' },
);
expect(noApp.status).toBe(403);
const scoped = await appIssuedScopedToken({
uid: theirs.uid,
token: mineIn,
});
const refused = await call('POST', '/events/workers/destroy', scoped, {
scope: 'user',
appUid: theirs.uid,
});
expect(refused.status).toBe(403);
});
it('works with user handlers switched off, so a user can stop paying', async () => {
const theirs = await makeApp(otherUserId);
await publishUserScope(await sessionIn(theirs, 'user'), 'x', SOURCE);
const config = (
env.server.services.events as unknown as {
config: { events: { userHandlers?: boolean } };
}
).config;
config.events.userHandlers = false;
try {
const destroyed = await call(
'POST',
'/events/workers/destroy',
env.users.user.token,
{ scope: 'user', appUid: theirs.uid },
);
expect(destroyed.status).toBe(200);
} finally {
config.events.userHandlers = true;
}
});
});
-5
View File
@@ -36,7 +36,6 @@ import { FSService } from './fs/FSService';
import { KvShareService } from './kvShare/KvShareService';
import { ServerHealthService } from './health/ServerHealthService';
import { PuterHomepageService } from './homepage/PuterHomepageService';
import { JobScheduler } from './jobs/JobScheduler';
import { LocalWorkerService } from './localworker/LocalWorkerService';
import { MeteringService } from './metering/MeteringService';
import { NotificationService } from './notification/NotificationService';
@@ -61,7 +60,6 @@ import { UserAccountService } from './user/UserAccountService';
*/
declare module './types' {
interface IPuterServiceInstances {
jobs: JobScheduler;
metering: MeteringService;
appOriginBlocklist: AppOriginBlocklistService;
permission: PermissionService;
@@ -105,9 +103,6 @@ declare module './types' {
// NotificationService depends on notification store (for DB) + event client (for socket push).
// BroadcastService is independent — only needs the event client.
export const puterServices = {
// First, so every later service can register jobs with it. Reaches
// `broadcast` at call time only.
jobs: JobScheduler,
metering: MeteringService,
// Declared before `auth` so AuthService sees it as a prior peer — it
// queries the blocklist on app-token acquisition and per-request app
@@ -1,781 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { randomUUID } from 'node:crypto';
import MockRedis from 'ioredis-mock';
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import type { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import {
JobLeaseLostError,
JobScheduler,
type JobRunContext,
} from './JobScheduler.js';
type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
const HOUR = 60 * 60_000;
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
describe('JobScheduler', () => {
let server: PuterServer;
const schedulers: JobScheduler[] = [];
beforeAll(async () => {
server = await setupTestServer();
});
afterAll(async () => {
await server?.shutdown();
});
afterEach(() => {
vi.restoreAllMocks();
vi.useRealTimers();
for (const scheduler of schedulers.splice(0)) {
scheduler.onServerPrepareShutdown();
}
});
const db = () => server.clients.db;
/**
* A node of its own: its own config and region, on the test server's Redis
* unless given one.
*/
const node = ({
namespace,
region = 'r1',
redis,
domain = 'puter.test',
enabled,
}: {
namespace?: string;
region?: string;
redis?: InstanceType<typeof MockRedis>;
domain?: string;
enabled?: boolean;
} = {}) => {
const config = {
port: 0,
extensions: [],
domain,
serverId: `node-${schedulers.length}`,
jobs: { namespace, enabled },
} as IConfig;
const clients = (
redis ? { ...server.clients, redis } : server.clients
) as SchedulerArgs[1];
const services = {
broadcast: { regionId: region },
} as unknown as SchedulerArgs[3];
const scheduler = new JobScheduler(
config,
clients,
server.stores as SchedulerArgs[2],
services,
);
schedulers.push(scheduler);
return scheduler;
};
const freshNamespace = () => `ns-${randomUUID()}`;
/**
* A job that runs `batches` batches `batchMs` apart, checking its signal
* before each.
*/
const batchedJob = (batches: number, batchMs: number) => {
const seen = { done: 0, signal: null as AbortSignal | null };
const run = async ({ signal }: JobRunContext) => {
seen.signal = signal;
for (let i = 0; i < batches; i++) {
if (signal.aborted) return;
await sleep(batchMs);
seen.done += 1;
}
};
return { seen, run };
};
const keysOf = (namespace: string, name: string) => ({
lock: `jobs:{${namespace}:${name}}:lock`,
gate: `jobs:{${namespace}:${name}}:next`,
run: `jobs:{${namespace}:${name}}:run`,
});
// -- Registration -------------------------------------------------
it('refuses a bad or duplicate registration', () => {
const scheduler = node({ namespace: freshNamespace() });
const run = async () => {};
expect(() =>
scheduler.register('bad name', {
every: 1000,
scope: 'fleet',
run,
}),
).toThrow(/invalid job name/);
expect(() =>
scheduler.register('job', { every: 0, scope: 'fleet', run }),
).toThrow(/every/);
expect(() =>
scheduler.register('job', {
every: 1000,
scope: 'fleet',
leaseMs: 1.5,
run,
}),
).toThrow(/leaseMs/);
// `:` would let two (namespace, name) pairs share Redis keys.
expect(() =>
scheduler.register('b:c', { every: 1000, scope: 'fleet', run }),
).toThrow(/invalid job name/);
expect(() =>
scheduler.register('x'.repeat(129), {
every: 1000,
scope: 'fleet',
run,
}),
).toThrow(/invalid job name/);
expect(() =>
scheduler.register('job.region', {
every: 1000,
scope: 'region',
leaseMs: 2000,
run,
}),
).toThrow(/exceeds every/);
scheduler.register('job', { every: 1000, scope: 'fleet', run });
expect(() =>
scheduler.register('job', { every: 1000, scope: 'fleet', run }),
).toThrow(/already registered/);
});
it('falls back to the default namespace, once and loudly, when the configured one cannot be held', () => {
const run = async () => {};
const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
for (const namespace of ['a:b', 'x'.repeat(129), 'spaced name']) {
const scheduler = node({ namespace });
scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
expect(scheduler.namespace).toBe('default');
}
logged.mockClear();
// A self-host whose domain carries a port still boots.
const scheduler = node({ domain: 'localhost:4100' });
scheduler.register('job.a', { every: 1000, scope: 'fleet', run });
scheduler.register('job.b', { every: 1000, scope: 'fleet', run });
expect(scheduler.namespace).toBe('default');
expect(logged).toHaveBeenCalledTimes(1);
expect(String(logged.mock.calls[0][0])).toMatch(
/localhost:4100.*jobs\.namespace/,
);
});
it('does nothing at all while no job is registered', async () => {
vi.useFakeTimers({
toFake: [
'setInterval',
'setTimeout',
'clearInterval',
'clearTimeout',
],
});
// Timers the test runtime itself already holds.
const baseline = vi.getTimerCount();
const reads = vi.spyOn(db(), 'read');
const preads = vi.spyOn(db(), 'pread');
const writes = vi.spyOn(db(), 'write');
const scheduler = node({ namespace: freshNamespace() });
scheduler.onServerStart();
expect(vi.getTimerCount()).toBe(baseline);
await scheduler.poll();
expect(reads).not.toHaveBeenCalled();
expect(preads).not.toHaveBeenCalled();
expect(writes).not.toHaveBeenCalled();
// A job registered after start arms the poll; shutdown disarms it.
scheduler.register('late', {
every: HOUR,
scope: 'fleet',
run: async () => {},
});
expect(vi.getTimerCount()).toBeGreaterThan(baseline);
scheduler.onServerPrepareShutdown();
expect(vi.getTimerCount()).toBe(baseline);
});
it('claims nothing once draining, and leaves a running job its lease', async () => {
const scheduler = node({ namespace: `drain${Date.now()}` });
let release!: () => void;
let signal!: AbortSignal;
scheduler.register('long', {
every: HOUR,
scope: 'region',
run: (ctx) => {
signal = ctx.signal;
return new Promise<void>((resolve) => {
release = resolve;
});
},
});
const first = scheduler.poll();
await vi.waitFor(() => expect(release).toBeDefined());
const run = vi.fn(async () => {});
scheduler.register('other', { every: HOUR, scope: 'region', run });
scheduler.onServerDrainStart();
await scheduler.poll();
expect(run).not.toHaveBeenCalled();
expect(signal.aborted).toBe(false);
release();
await first;
});
it('arms no poll on a node with jobs turned off, though poll() still runs them', async () => {
vi.useFakeTimers({
toFake: [
'setInterval',
'setTimeout',
'clearInterval',
'clearTimeout',
],
});
const baseline = vi.getTimerCount();
const scheduler = node({ namespace: freshNamespace(), enabled: false });
const warned = vi.spyOn(console, 'warn').mockImplementation(() => {});
let runs = 0;
scheduler.onServerStart();
scheduler.register('job.off', {
every: HOUR,
scope: 'fleet',
run: async () => {
runs += 1;
},
});
scheduler.register('job.off.too', {
every: HOUR,
scope: 'region',
run: async () => undefined,
});
expect(vi.getTimerCount()).toBe(baseline);
// Said once per node, not per job.
expect(
warned.mock.calls.filter(([line]) =>
String(line).includes('jobs.enabled'),
),
).toHaveLength(1);
vi.useRealTimers();
await scheduler.poll();
expect(runs).toBe(1);
});
describe('on a controlled clock', () => {
const leaseMs = 300;
/**
* Pending timeouts armed from JobScheduler.ts. The shared test server's
* own loops keep running on real intervals and may arm fake timers
* mid-test, so the global timer count says nothing about the
* scheduler.
*/
let armed: Set<unknown>;
const schedulerTimers = () => armed.size;
beforeEach(() => {
vi.useFakeTimers({
toFake: [
'setTimeout',
'clearTimeout',
'setInterval',
'clearInterval',
'performance',
],
});
armed = new Set();
const fakeSetTimeout = globalThis.setTimeout;
const fakeClearTimeout = globalThis.clearTimeout;
const fromScheduler = () => {
const limit = Error.stackTraceLimit;
Error.stackTraceLimit = 30;
const stack = new Error().stack ?? '';
Error.stackTraceLimit = limit;
return /services[\\/]jobs[\\/]JobScheduler\.ts/.test(stack);
};
vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
fn: (...args: unknown[]) => void,
ms?: number,
...rest: unknown[]
) => {
if (!fromScheduler()) return fakeSetTimeout(fn, ms, ...rest);
const handle = fakeSetTimeout(
(...args: unknown[]) => {
armed.delete(handle);
fn(...args);
},
ms,
...rest,
);
armed.add(handle);
return handle;
}) as never);
vi.spyOn(globalThis, 'clearTimeout').mockImplementation(((
handle: Parameters<typeof clearTimeout>[0],
) => {
armed.delete(handle);
fakeClearTimeout(handle);
}) as never);
});
/** Batches of `batchMs`, each started only while the signal is live. */
const timedJob = (batches: number, batchMs: number) => {
const seen = {
starts: [] as number[],
abortedAt: null as number | null,
signal: null as AbortSignal | null,
saveError: null as unknown,
};
const run = async ({ signal, saveState }: JobRunContext) => {
seen.signal = signal;
signal.addEventListener('abort', () => {
seen.abortedAt = performance.now();
});
for (let i = 0; i < batches; i++) {
if (signal.aborted) break;
seen.starts.push(performance.now());
await sleep(batchMs);
}
try {
await saveState({ late: true });
} catch (err) {
seen.saveError = err;
}
};
return { seen, run };
};
/** Let the run play out, then check nothing it armed is left behind. */
const settle = async (running: Promise<void>) => {
await vi.advanceTimersByTimeAsync(2_000);
await running;
expect(schedulerTimers()).toBe(0);
};
it('clears a run’s timers at shutdown even while the run hangs', async () => {
const scheduler = node({ namespace: freshNamespace() });
let release!: () => void;
scheduler.register('job.hang', {
every: HOUR,
scope: 'region',
leaseMs,
run: () =>
new Promise<void>((resolve) => {
release = resolve;
}),
});
const running = scheduler.poll();
await vi.advanceTimersByTimeAsync(10);
expect(schedulerTimers()).toBeGreaterThan(0);
scheduler.onServerPrepareShutdown();
expect(schedulerTimers()).toBe(0);
release();
await settle(running);
});
it('never starts a region run once taking the lock used up half the lease', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17104 });
const scheduler = node({ namespace, redis });
const run = vi.fn(async () => {});
scheduler.register('job.slowLock', {
every: HOUR,
scope: 'region',
leaseMs,
run,
});
const set = redis.set.bind(redis) as (
...args: unknown[]
) => Promise<unknown>;
const slowLock = vi.spyOn(redis, 'set').mockImplementation((async (
...args: unknown[]
) => {
const result = await set(...args);
if (String(args[0]).endsWith(':lock')) {
await sleep(leaseMs / 2 + 10);
}
return result;
}) as never);
await settle(scheduler.poll());
expect(run).not.toHaveBeenCalled();
// The gate went back, so the next poll runs it.
expect(
await redis.exists(`jobs:{${namespace}:job.slowLock}:next`),
).toBe(0);
slowLock.mockRestore();
await settle(scheduler.poll());
expect(run).toHaveBeenCalledTimes(1);
});
it('aborts a region run before its next batch once the lock lapses, and leaves it due at once', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17106 });
const scheduler = node({ namespace, redis });
const job = timedJob(10, 70);
scheduler.register('job.lapse', {
every: HOUR,
scope: 'region',
leaseMs,
run: job.run,
});
const lockedAt = performance.now();
await settle(scheduler.poll());
expect(job.seen.abortedAt! - lockedAt).toBe(leaseMs);
expect((job.seen.signal?.reason as Error).message).toMatch(
/lapsed/,
);
expect(job.seen.starts.length).toBeLessThan(10);
expect(job.seen.starts.every((t) => t < job.seen.abortedAt!)).toBe(
true,
);
// The gate is still this run's, so its last save went through;
// interrupted, the gate goes back and the job is due at once.
expect(job.seen.saveError).toBeNull();
const keys = {
gate: `jobs:{${namespace}:job.lapse}:next`,
run: `jobs:{${namespace}:job.lapse}:run`,
};
expect(await redis.exists(keys.gate)).toBe(0);
expect(await redis.hget(keys.run, 'state')).toBe('{"late":true}');
expect(await redis.hget(keys.run, 'failures')).toBeNull();
});
});
it('refuses state over 64 KB', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17107 });
const scheduler = node({ namespace, redis });
let refused: unknown = null;
scheduler.register('job.bigState', {
every: HOUR,
scope: 'region',
run: async ({ saveState }) => {
try {
await saveState({ blob: 'x'.repeat(70_000) });
} catch (err) {
refused = err;
}
},
});
await scheduler.poll();
expect((refused as Error).message).toMatch(/over 65536 bytes/);
expect(
await redis.hget(keysOf(namespace, 'job.bigState').run, 'state'),
).toBeNull();
});
it('releases an interrupted run on shutdown without counting a failure', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17109 });
const scheduler = node({ namespace, redis });
const job = batchedJob(50, 50);
scheduler.register('job.shutdown', {
every: HOUR,
scope: 'region',
run: job.run,
});
const running = scheduler.poll();
await vi.waitFor(() => expect(job.seen.done).toBeGreaterThan(0));
scheduler.onServerPrepareShutdown();
await running;
expect(job.seen.done).toBeLessThan(50);
expect((job.seen.signal?.reason as Error).message).toBe(
'server shutting down',
);
const keys = keysOf(namespace, 'job.shutdown');
// Due again at once, with no failure counted.
expect(await redis.exists(keys.gate)).toBe(0);
expect(await redis.hget(keys.run, 'failures')).toBeNull();
});
it('keeps namespaces apart: the same job name runs once in each', async () => {
const redis = new MockRedis({ port: 17110 });
const [nsA, nsB] = [freshNamespace(), freshNamespace()];
const nodes = [
node({ namespace: nsA, redis }),
node({ namespace: nsA, redis }),
node({ namespace: nsB, redis }),
node({ namespace: nsB, redis }),
];
const runs: string[] = [];
nodes.forEach((scheduler, i) =>
scheduler.register('job.shared', {
every: HOUR,
scope: 'region',
run: async () => {
runs.push(i < 2 ? 'a' : 'b');
await sleep(20);
},
}),
);
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
expect(runs.sort()).toEqual(['a', 'b']);
});
it('falls back to the domain as its namespace', () => {
expect(node({ domain: 'example.test' }).namespace).toBe('example.test');
expect(
node({ namespace: 'explicit', domain: 'example.test' }).namespace,
).toBe('explicit');
});
it('boots a server whose domain carries a port, registering its jobs under the default namespace', async () => {
const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
const portDomain = await setupTestServer({
domain: 'localhost:4101',
} as IConfig);
try {
expect(portDomain.services.jobs.namespace).toBe('default');
const namespaceErrors = logged.mock.calls.filter(([message]) =>
String(message).startsWith('[jobs] namespace'),
);
expect(namespaceErrors).toHaveLength(1);
} finally {
await portDomain.shutdown();
}
});
// -- Region scope -------------------------------------------------
it('runs a region job once per region', async () => {
const namespace = freshNamespace();
const redisA = new MockRedis({ port: 17101 });
const redisB = new MockRedis({ port: 17102 });
const nodes = [
node({ namespace, region: 'a', redis: redisA }),
node({ namespace, region: 'a', redis: redisA }),
node({ namespace, region: 'b', redis: redisB }),
node({ namespace, region: 'b', redis: redisB }),
];
const runs: string[] = [];
nodes.forEach((scheduler, i) =>
scheduler.register('job.region', {
every: HOUR,
scope: 'region',
leaseMs: 5_000,
run: async () => {
runs.push(i < 2 ? 'a' : 'b');
await sleep(20);
},
}),
);
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
expect(runs.sort()).toEqual(['a', 'b']);
expect(
await redisA.pttl(`jobs:{${namespace}:job.region}:next`),
).toBeGreaterThan(HOUR - 60_000);
// The region lock was released after the run.
expect(await redisA.exists(`jobs:{${namespace}:job.region}:lock`)).toBe(
0,
);
});
it('backs off a failing region job and keeps its state in Redis', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17103 });
const scheduler = node({ namespace, redis });
const seen: unknown[] = [];
let fail = true;
scheduler.register('job.regionFail', {
every: HOUR,
scope: 'region',
leaseMs: 5_000,
run: async ({ state, saveState }) => {
seen.push(state);
await saveState({ page: seen.length });
if (fail) throw new Error('nope');
},
});
const gate = `jobs:{${namespace}:job.regionFail}:next`;
await scheduler.poll();
expect(
await redis.hget(
`jobs:{${namespace}:job.regionFail}:run`,
'failures',
),
).toBe('1');
expect(await redis.pttl(gate)).toBeLessThanOrEqual(2_000);
fail = false;
await redis.del(gate);
await scheduler.poll();
expect(seen).toEqual([null, { page: 1 }]);
expect(
await redis.hget(
`jobs:{${namespace}:job.regionFail}:run`,
'failures',
),
).toBeNull();
expect(await redis.pttl(gate)).toBeGreaterThan(HOUR - 60_000);
});
it('skips region jobs quietly while Redis is unreachable', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17108 });
const scheduler = node({ namespace, redis });
const run = vi.fn(async () => {});
scheduler.register('job.noRedis', {
every: HOUR,
scope: 'region',
run,
});
vi.spyOn(redis, 'exists').mockRejectedValue(
new Error('connection refused'),
);
const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
await scheduler.poll();
expect(run).not.toHaveBeenCalled();
expect(errors).not.toHaveBeenCalled();
});
it('fences a region run that outlived its gate off its successor’s keys', async () => {
const namespace = freshNamespace();
const redis = new MockRedis({ port: 17105 });
const [slow, next] = [
node({ namespace, redis }),
node({ namespace, redis }),
];
const keys = {
gate: `jobs:{${namespace}:job.fenced}:next`,
run: `jobs:{${namespace}:job.fenced}:run`,
};
let release!: () => void;
let slowStarted = false;
let slowSaveError: unknown = null;
// Ignores its signal and outlives both its 200 ms lock and 400 ms gate.
slow.register('job.fenced', {
every: 400,
scope: 'region',
leaseMs: 200,
run: async ({ saveState }) => {
slowStarted = true;
await new Promise<void>((resolve) => {
release = resolve;
});
try {
await saveState({ by: 'slow' });
} catch (err) {
slowSaveError = err;
}
throw new Error('slow run failed');
},
});
next.register('job.fenced', {
every: 400,
scope: 'region',
leaseMs: 200,
run: async ({ saveState }) => {
await saveState({ by: 'next' });
},
});
const slowRun = slow.poll();
await vi.waitFor(() => expect(slowStarted).toBe(true));
await sleep(450);
await next.poll();
const successorGate = await redis.get(keys.gate);
expect(successorGate).not.toBeNull();
release();
await slowRun;
expect(slowSaveError).toBeInstanceOf(JobLeaseLostError);
// Neither the slow run's state, its failure count, nor its cleanup
// touched what the successor holds.
expect(await redis.get(keys.gate)).toBe(successorGate);
expect(await redis.pttl(keys.gate)).toBeGreaterThan(0);
expect(await redis.hget(keys.run, 'state')).toBe('{"by":"next"}');
expect(await redis.hget(keys.run, 'failures')).toBeNull();
});
it('runs a fleet job once per region, its lease capped at its cadence and nearPrimary ignored', async () => {
const namespace = freshNamespace();
const redisA = new MockRedis({ port: 17111 });
const redisB = new MockRedis({ port: 17112 });
const nodes = [
node({ namespace, region: 'a', redis: redisA }),
node({ namespace, region: 'a', redis: redisA }),
node({ namespace, region: 'b', redis: redisB }),
node({ namespace, region: 'b', redis: redisB }),
];
const keys = keysOf(namespace, 'job.fleet');
const runs: string[] = [];
const lockTtls: number[] = [];
const reads = vi.spyOn(db(), 'read');
const preads = vi.spyOn(db(), 'pread');
const writes = vi.spyOn(db(), 'write');
nodes.forEach((scheduler, i) => {
const redis = i < 2 ? redisA : redisB;
scheduler.register('job.fleet', {
every: 2_000,
scope: 'fleet',
nearPrimary: true,
// Longer than `every`: refused for a region job, capped here.
leaseMs: HOUR,
run: async () => {
runs.push(i < 2 ? 'a' : 'b');
lockTtls.push(await redis.pttl(keys.lock));
await sleep(20);
},
});
});
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
expect(runs.sort()).toEqual(['a', 'b']);
for (const ttl of lockTtls) {
expect(ttl).toBeGreaterThan(0);
expect(ttl).toBeLessThanOrEqual(2_000);
}
expect(await redisA.pttl(keys.gate)).toBeGreaterThan(1_000);
expect(await redisB.pttl(keys.gate)).toBeGreaterThan(1_000);
expect(reads).not.toHaveBeenCalled();
expect(preads).not.toHaveBeenCalled();
expect(writes).not.toHaveBeenCalled();
});
});
-528
View File
@@ -1,528 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { randomUUID } from 'node:crypto';
import { withRedisLock } from '../../util/redisLock.js';
import { resolveJobsNamespace } from '../../util/jobsNamespace.js';
import { deliveryBackoffMs } from '../../util/retryBackoff.js';
import { PuterService } from '../types';
import {
JobLeaseLostError,
asError,
lostLease,
parseState,
runOutcome,
serializeState,
type RegisteredJob,
type RunLease,
} from './jobRun.js';
export { JobLeaseLostError } from './jobRun.js';
/** How often each node looks for due jobs. */
const JOB_POLL_MS = 30_000;
/** Each poll waits up to this long first, so nodes don't all claim at once. */
const JOB_POLL_JITTER_MS = 5_000;
const JOB_DEFAULT_LEASE_MS = 15 * 60_000;
/** Consecutive failures that raise `scheduled_job_failing`. */
const JOB_FAILING_ALARM_AT = 3;
const JOB_SHUTDOWN_WAIT_MS = 10_000;
/**
* Job names: ASCII, at most 128, and without `:`, so a (namespace, name) pair
* maps to exactly one set of Redis keys (namespaces follow the same rule).
*/
const JOB_ID_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
// Region writes apply only while the gate still holds this run's token.
const REGION_FENCED_WRITE = `
if redis.call('get', KEYS[1]) ~= ARGV[1] then return -1 end
local op = ARGV[2]
if op == 'hset' then return redis.call('hset', KEYS[2], ARGV[3], ARGV[4]) end
if op == 'hdel' then return redis.call('hdel', KEYS[2], ARGV[3]) end
if op == 'hincrby' then return redis.call('hincrby', KEYS[2], ARGV[3], 1) end
if op == 'pexpire' then return redis.call('pexpire', KEYS[1], ARGV[3]) end
if op == 'del' then return redis.call('del', KEYS[1]) end
return -1
`;
/**
* `region` runs once per cadence per region. `fleet` means once across every
* region where `services.jobs` supports it; this default runs it per region.
*/
export type JobScope = 'fleet' | 'region';
export interface JobRunContext {
/** Aborts once the run may have lost the job. Check it before each batch. */
signal: AbortSignal;
/** What an earlier run saved, or null. */
state: unknown;
/**
* Persist JSON-serializable `state` for later runs. Throws once the run
* lost the job.
*/
saveState: (state: unknown) => Promise<void>;
}
export interface JobDefinition {
/** Cadence in ms. */
every: number;
scope: JobScope;
/**
* Prefer the region nearest the primary database, for a `fleet` job. The
* default scheduler ignores it.
*/
nearPrimary?: boolean;
/**
* How long a run holds the job, in total on the region path (at most
* `every`). Default 15 min, capped at `every` for `region`; a `fleet` job
* run per region is capped at `every` too.
*/
leaseMs?: number;
run: (ctx: JobRunContext) => Promise<void>;
}
/** The `afterId` cursor a keyset-walking job saved in its state, or 0. */
export const savedAfterId = (state: unknown): number => {
const afterId = (state as { afterId?: unknown } | null)?.afterId;
return typeof afterId === 'number' &&
Number.isSafeInteger(afterId) &&
afterId > 0
? afterId
: 0;
};
interface RegionKeys {
lock: string;
gate: string;
run: string;
}
/**
* Runs our own recurring backend work once per cadence per region, under a
* Redis lock. `fleet` jobs take the same path here; an extension registered as
* `services.jobs` can run them fleet-wide by overriding `pollFleet`. Nothing
* polls until a job is registered.
*/
export class JobScheduler extends PuterService {
#jobs = new Map<string, RegisteredJob>();
#running = new Map<string, Promise<void>>();
#leases = new Set<RunLease>();
#started = false;
#stopping = false;
/** Polling stopped; runs already going keep their leases. */
#draining = false;
#timer: ReturnType<typeof setInterval> | null = null;
#pollDelays = new Set<ReturnType<typeof setTimeout>>();
#warnedOff = false;
#namespace: string | null = null;
/** Keys this deployment's jobs, so deployments sharing Redis stay apart. */
get namespace(): string {
return (this.#namespace ??= resolveJobsNamespace(this.config));
}
register(name: string, definition: JobDefinition): void {
if (!JOB_ID_PATTERN.test(name)) {
throw new Error(`invalid job name: ${name}`);
}
if (this.#jobs.has(name)) {
throw new Error(`job already registered: ${name}`);
}
const { every, scope } = definition;
if (!Number.isSafeInteger(every) || every <= 0) {
throw new Error(`job ${name}: every must be a positive integer`);
}
if (scope !== 'fleet' && scope !== 'region') {
throw new Error(`job ${name}: unknown scope`);
}
const leaseMs =
definition.leaseMs ??
(scope === 'region'
? Math.min(JOB_DEFAULT_LEASE_MS, every)
: JOB_DEFAULT_LEASE_MS);
if (!Number.isSafeInteger(leaseMs) || leaseMs <= 0) {
throw new Error(`job ${name}: leaseMs must be a positive integer`);
}
// The gate that fences a region run lasts `every`.
if (scope === 'region' && leaseMs > every) {
throw new Error(
`job ${name}: a region job's leaseMs exceeds every`,
);
}
this.#jobs.set(name, {
every,
scope,
nearPrimary: definition.nearPrimary ?? false,
leaseMs,
run: definition.run,
});
if (this.#started) this.#arm();
}
override onServerStart(): void {
this.#started = true;
if (this.#jobs.size > 0) this.#arm();
}
/** Claim nothing new while draining; running jobs carry on until prepare. */
override onServerDrainStart(): void {
this.#stopPolling();
}
override onServerPrepareShutdown(): void {
this.#stopping = true;
this.#stopPolling();
for (const lease of [...this.#leases]) {
lease.controller.abort(
new JobLeaseLostError('server shutting down'),
);
lease.close();
}
}
#stopPolling(): void {
this.#draining = true;
if (this.#timer) clearInterval(this.#timer);
this.#timer = null;
for (const delay of this.#pollDelays) clearTimeout(delay);
this.#pollDelays.clear();
}
/**
* Give aborted runs a moment to record themselves before the database
* closes.
*/
override async onServerShutdown(): Promise<void> {
if (this.#running.size === 0) return;
let wait: ReturnType<typeof setTimeout> | undefined;
await Promise.race([
Promise.allSettled(this.#running.values()),
new Promise((resolve) => {
wait = setTimeout(resolve, JOB_SHUTDOWN_WAIT_MS);
}),
]);
clearTimeout(wait);
}
/**
* Start every registered job that is due and not already running here.
* Resolves once the runs it started have finished.
*/
async poll(): Promise<void> {
if (this.#stopping || this.#draining) return;
const fleet: string[] = [];
const region: Promise<void>[] = [];
for (const [name, job] of this.#jobs) {
if (this.#running.has(name)) continue;
if (job.scope === 'fleet') {
fleet.push(name);
continue;
}
region.push(this.track(name, () => this.#tryRegion(name, job)));
}
await Promise.all([this.pollFleet(fleet), ...region]);
}
#arm(): void {
if (this.#timer || this.#stopping || this.#draining) return;
if (this.config.jobs?.enabled === false) {
// Nothing alarms on a job that never runs, so say it here.
if (!this.#warnedOff) {
console.warn(
'[jobs] polling is off on this node (jobs.enabled: false); other nodes must run the jobs',
);
}
this.#warnedOff = true;
return;
}
const tick = () => {
const delay = setTimeout(() => {
this.#pollDelays.delete(delay);
void this.#pollSafely();
}, Math.random() * JOB_POLL_JITTER_MS);
delay.unref?.();
this.#pollDelays.add(delay);
};
const timer = setInterval(tick, JOB_POLL_MS);
timer.unref?.();
this.#timer = timer;
tick();
}
async #pollSafely(): Promise<void> {
try {
await this.poll();
} catch (err) {
console.error('[jobs] poll failed', err);
}
}
// -- Region scope -------------------------------------------------
async #tryRegion(name: string, job: RegisteredJob): Promise<void> {
const redis = this.clients.redis;
const keys = regionKeys(this.namespace, name);
let gated: number;
try {
gated = await redis.exists(keys.gate);
} catch {
// Redis is unreachable; the lock below would be busy anyway.
return;
}
if (gated > 0) return;
// Before the lock, so acquiring it counts against `leaseMs`.
const startedAt = performance.now();
await withRedisLock(
redis,
keys.lock,
async () => {
const token = randomUUID();
const gate = await redis.set(
keys.gate,
token,
'PX',
job.every,
'NX',
);
if (gate !== 'OK') return;
await this.#runRegion(name, job, keys, token, startedAt);
},
{
ttlMs: job.leaseMs,
onUnavailable: 'busy',
onBusy: (): void => undefined,
},
);
}
/**
* One run under the region lock. The lock doesn't renew, so the signal
* aborts once it lapses, and every write is fenced on the gate's token.
*/
async #runRegion(
name: string,
job: RegisteredJob,
keys: RegionKeys,
token: string,
startedAt: number,
): Promise<void> {
const lease = this.#regionLease(job.leaseMs, startedAt);
const { signal } = lease.controller;
const write = (op: string, ...args: (string | number)[]) =>
this.#regionWrite(keys, token, op, ...args);
let error: unknown = null;
try {
const saved = await this.clients.redis.hget(keys.run, 'state');
if (await lease.ready()) {
await job.run({
signal,
state: parseState(saved),
saveState: async (state) => {
const json = serializeState(state);
if ((await write('hset', 'state', json)) < 0) {
lostLease(lease, name);
}
},
});
}
} catch (err) {
error = err;
} finally {
lease.close();
}
const outcome = runOutcome(signal, error);
if (outcome === 'success') {
await write('hdel', 'failures');
} else if (outcome === 'failure') {
const failures = await write('hincrby', 'failures');
if (failures < 0) return;
await write(
'pexpire',
Math.min(job.every, deliveryBackoffMs(failures)),
);
this.reportFailure(name, failures, error);
} else {
await write('del');
}
}
/**
* The lock lapses `leaseMs` after `from`; the run needs half of it left to
* start.
*/
#regionLease(leaseMs: number, from: number): RunLease {
const controller = new AbortController();
const deadline = from + leaseMs;
const abort = (reason: string) => {
if (!controller.signal.aborted) {
controller.abort(new JobLeaseLostError(reason));
}
};
const lapse = setTimeout(
() => abort('region lock lapsed'),
Math.max(0, deadline - performance.now()),
);
const lease: RunLease = {
controller,
ready: async () => {
if (
!controller.signal.aborted &&
deadline - performance.now() >= leaseMs / 2
) {
return true;
}
abort('too little of the lock left to start');
return false;
},
close: () => {
clearTimeout(lapse);
this.closeLease(lease);
},
};
this.openLease(lease);
return lease;
}
/**
* Run a write to the region keys if the gate still holds `token`; -1 if
* not.
*/
async #regionWrite(
keys: RegionKeys,
token: string,
op: string,
...args: (string | number)[]
): Promise<number> {
const result = await this.clients.redis.eval(
REGION_FENCED_WRITE,
2,
keys.gate,
keys.run,
token,
op,
...args,
);
return Number(result);
}
// -- Hooks, also for extensions that replace `services.jobs` -------
/**
* Run `fleet` jobs registered here and not running in this process. The
* default holds each per region, its lease capped at `every`.
*/
protected async pollFleet(names: string[]): Promise<void> {
await Promise.all(
names.map((name) => {
const job = this.definitionOf(name);
return this.track(name, () =>
this.#tryRegion(name, {
...job,
leaseMs: Math.min(job.leaseMs, job.every),
}),
);
}),
);
}
/** Runs `work` as `name`'s only run in this process. */
protected track(name: string, work: () => Promise<void>): Promise<void> {
const run = (async () => {
try {
await work();
} catch (err) {
console.error(`[jobs] ${name} errored`, err);
} finally {
this.#running.delete(name);
}
})();
this.#running.set(name, run);
return run;
}
protected isRunning(name: string): boolean {
return this.#running.has(name);
}
protected definitionOf(name: string): RegisteredJob {
const job = this.#jobs.get(name);
if (!job) throw new Error(`job not registered: ${name}`);
return job;
}
/** Who runs a job here: region, server and process. */
protected owner(): string {
const region = this.services.broadcast.regionId;
const server = this.config.serverId ?? 'local';
return `${region}:${server}:${process.pid}`.slice(0, 128);
}
/**
* Track `lease` for shutdown; one opened while shutting down is aborted at
* once.
*/
protected openLease(lease: RunLease): void {
this.#leases.add(lease);
if (this.#stopping) {
lease.controller.abort(
new JobLeaseLostError('server shutting down'),
);
}
}
/** Stop tracking `lease`; its `close` calls this. */
protected closeLease(lease: RunLease): void {
this.#leases.delete(lease);
}
protected reportFailure(
name: string,
failures: number,
error: unknown,
): void {
console.error(`[jobs] ${name} failed (${failures} in a row)`, error);
if (failures < JOB_FAILING_ALARM_AT) return;
this.clients.alarm.create(
`scheduled_job_failing:${this.namespace}:${name}`,
`System job ${name} keeps failing`,
{
namespace: this.namespace,
name,
failures,
error: asError(error),
},
'warning',
{ dedup: true },
);
}
}
/**
* One hash slot per job. Neither part may contain `:` (`JOB_ID_PATTERN`), so no
* two (namespace, name) pairs share keys.
*/
const regionKeys = (namespace: string, name: string): RegionKeys => {
const tag = `{${namespace}:${name}}`;
return {
lock: `jobs:${tag}:lock`,
gate: `jobs:${tag}:next`,
run: `jobs:${tag}:run`,
};
};
-82
View File
@@ -1,82 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// What one job run needs, whichever scheduler holds the job. Also used by
// extensions that replace `services.jobs`.
import type { JobDefinition } from './JobScheduler.js';
export const JOB_STATE_MAX_BYTES = 64 * 1024;
export const JOB_ERROR_MAX_CHARS = 1024;
/** The run no longer holds its job; another node may already be running it. */
export class JobLeaseLostError extends Error {}
export type RegisteredJob = Required<JobDefinition>;
export type RunOutcome = 'success' | 'failure' | 'interrupted';
/** What a run holds the job by. Its deadline is measured from before the claim. */
export interface RunLease {
controller: AbortController;
/**
* Renews if little lease is left; false (and aborted) if the run can't
* start.
*/
ready: () => Promise<boolean>;
close: () => void;
}
export const lostLease = (lease: RunLease, name: string): never => {
const lost = new JobLeaseLostError(`job ${name} lost its lease`);
lease.controller.abort(lost);
throw lost;
};
/** A run cut short by its signal is interrupted, whether or not it threw. */
export const runOutcome = (signal: AbortSignal, error: unknown): RunOutcome => {
if (signal.aborted) return 'interrupted';
return error === null ? 'success' : 'failure';
};
export const parseState = (raw: string | null): unknown => {
if (raw === null) return null;
try {
return JSON.parse(raw);
} catch {
return null;
}
};
export const serializeState = (state: unknown): string => {
const json = JSON.stringify(state ?? null);
if (Buffer.byteLength(json) > JOB_STATE_MAX_BYTES) {
throw new Error(`job state is over ${JOB_STATE_MAX_BYTES} bytes`);
}
return json;
};
export const asError = (error: unknown): Error =>
error instanceof Error ? error : new Error(String(error));
export const errorText = (error: unknown): string =>
(error instanceof Error ? error.message : String(error)).slice(
0,
JOB_ERROR_MAX_CHARS,
);
@@ -1,273 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The backend's own periodic jobs on the default scheduler: what registers,
* that each runs once across nodes sharing Redis, and that a single self-hosted
* node (sqlite, in-process Redis) runs every one of them.
*/
import { randomUUID } from 'node:crypto';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import type { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import {
JobScheduler,
type JobDefinition,
type JobRunContext,
} from './JobScheduler.js';
type SchedulerArgs = ConstructorParameters<typeof JobScheduler>;
const MINUTE = 60_000;
const HOUR = 60 * MINUTE;
const EXPECTED_JOBS = {
'auth.sessions.purge': {
every: 7 * 24 * HOUR,
scope: 'fleet',
nearPrimary: true,
leaseMs: 30 * MINUTE,
},
'kvShare.handles.purgeRevoked': {
every: 24 * HOUR,
scope: 'fleet',
nearPrimary: true,
leaseMs: 15 * MINUTE,
},
'notifications.retention': {
every: HOUR,
scope: 'fleet',
nearPrimary: true,
leaseMs: 15 * MINUTE,
},
'metering.globalRateCheck': {
every: 25 * MINUTE,
scope: 'fleet',
nearPrimary: false,
leaseMs: 5 * MINUTE,
},
'events.subscriptions.expirySweep': {
every: HOUR,
scope: 'fleet',
nearPrimary: true,
leaseMs: 15 * MINUTE,
},
'events.userHandlers.gc': {
every: HOUR,
scope: 'fleet',
nearPrimary: true,
leaseMs: 15 * MINUTE,
},
'events.subscriptions.creditSweep': {
every: 15 * MINUTE,
scope: 'fleet',
nearPrimary: false,
leaseMs: 10 * MINUTE,
},
};
const JOB_NAMES = Object.keys(EXPECTED_JOBS);
/** A timer armed while the server booted, and the frame that armed it. */
interface ArmedTimer {
kind: 'interval' | 'timeout';
ms: number | undefined;
caller: string;
}
let server: PuterServer;
const registered = new Map<string, JobDefinition>();
const armed: ArmedTimer[] = [];
/** The first stack frame outside this file and node_modules. */
const callerOf = (stack: string | undefined): string =>
(stack ?? '')
.split('\n')
.slice(1)
.find(
(line) =>
!line.includes('systemJobs.test') &&
!line.includes('node_modules'),
) ?? '';
beforeAll(async () => {
const register = vi.spyOn(JobScheduler.prototype, 'register');
const realInterval = globalThis.setInterval;
const realTimeout = globalThis.setTimeout;
const intervals = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
...args: Parameters<typeof setInterval>
) => {
armed.push({
kind: 'interval',
ms: args[1],
caller: callerOf(new Error().stack),
});
return realInterval(...args);
}) as never);
const timeouts = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
...args: Parameters<typeof setTimeout>
) => {
armed.push({
kind: 'timeout',
ms: args[1],
caller: callerOf(new Error().stack),
});
return realTimeout(...args);
}) as never);
try {
server = await setupTestServer({
events: { enabled: true },
notificationRetentionDays: 14,
maxGlobalUsagePerMinute: 1e15,
// A self-hosted node: nothing turns its jobs off.
jobs: { enabled: true },
} as IConfig);
} finally {
intervals.mockRestore();
timeouts.mockRestore();
}
for (const [name, definition] of register.mock.calls)
registered.set(name, definition);
register.mockRestore();
});
afterAll(async () => {
await server?.shutdown();
});
const keysOf = (namespace: string, name: string) => ({
lock: `jobs:{${namespace}:${name}}:lock`,
gate: `jobs:{${namespace}:${name}}:next`,
run: `jobs:{${namespace}:${name}}:run`,
});
/** A node of its own in `region`, on the test server's Redis. */
const node = (namespace: string, region: string): JobScheduler =>
new JobScheduler(
{
domain: 'puter.test',
serverId: `node-${region}`,
jobs: { namespace },
} as IConfig,
server.clients as SchedulerArgs[1],
server.stores as SchedulerArgs[2],
{
broadcast: { regionId: region },
} as unknown as SchedulerArgs[3],
);
describe('system jobs', () => {
it('registers every moved job with its real cadence, scope and lease', () => {
expect(
Object.fromEntries(
[...registered].map(([name, job]) => [
name,
{
every: job.every,
scope: job.scope,
nearPrimary: job.nearPrimary ?? false,
leaseMs: job.leaseMs,
},
]),
),
).toEqual(EXPECTED_JOBS);
});
it('leaves no per-node timer behind for a moved job', () => {
const armedFrom = (file: string) =>
armed
.filter(({ caller }) => caller.includes(file))
.map(({ kind, ms }) => ({ kind, ms }));
expect(armedFrom('services/auth/AuthService')).toEqual([]);
expect(armedFrom('services/notification/NotificationService')).toEqual(
[],
);
// Node-local: the usage buffer flush.
expect(armedFrom('services/metering/MeteringService')).toEqual([
{ kind: 'interval', ms: 10_000 },
]);
// Node-local: the share-job runner (DB claims per job), and its kick.
expect(armedFrom('services/kvShare/KvShareService')).toEqual([
{ kind: 'timeout', ms: expect.any(Number) },
{ kind: 'interval', ms: MINUTE },
]);
// Node-local: the region's pending-delivery sweep.
expect(armedFrom('services/events/EventsService')).toEqual([
{ kind: 'interval', ms: 10_000 },
]);
});
it.each(JOB_NAMES)(
'runs %s once across three nodes sharing Redis',
async (name) => {
const namespace = `ns-${randomUUID()}`;
const nodes = ['n1', 'n2', 'n3'].map((id) => node(namespace, id));
const job = registered.get(name)!;
let runs = 0;
for (const scheduler of nodes)
scheduler.register(name, {
...job,
run: async (ctx: JobRunContext) => {
runs += 1;
await job.run(ctx);
},
});
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
await Promise.all(nodes.map((scheduler) => scheduler.poll()));
expect(runs).toBe(1);
const redis = server.clients.redis;
const keys = keysOf(namespace, name);
// A failure would have cut the gate to a 2 s backoff.
expect(await redis.pttl(keys.gate)).toBeGreaterThan(
job.every - MINUTE,
);
expect(await redis.hget(keys.run, 'failures')).toBeNull();
},
);
it('runs every job on a single self-hosted node, without the old per-job gates', async () => {
const jobs = server.services.jobs;
const redis = server.clients.redis;
await jobs.poll();
// The node's own poll timer may have started some of them first.
await vi.waitFor(
async () => {
for (const name of JOB_NAMES) {
const keys = keysOf(jobs.namespace, name);
expect(await redis.exists(keys.gate), name).toBe(1);
// Released once the run is over.
expect(await redis.exists(keys.lock), name).toBe(0);
expect(await redis.hget(keys.run, 'failures'), name).toBe(
null,
);
}
},
{ timeout: 10_000, interval: 100 },
);
await expect(
redis.exists('sessions:purge:gate', 'kvShare:purge:gate'),
).resolves.toBe(0);
});
});
+69 -33
View File
@@ -371,9 +371,11 @@ const SHARE_CHANGE_LOCK_RETRY_MS = 50;
/** Retired handles are kept this long for their owner's listing, then deleted. */
export const KV_SHARE_REVOKED_RETENTION_SECONDS = 30 * 24 * 60 * 60;
const PURGE_JOB = 'kvShare.handles.purgeRevoked';
const PURGE_EVERY_MS = 24 * 60 * 60 * 1000;
const PURGE_LEASE_MS = 15 * 60 * 1000;
// Every node checks hourly; the gate lets one node through per day.
const PURGE_CHECK_MS = 60 * 60 * 1000;
const PURGE_INITIAL_DELAY_MS = 5 * 60 * 1000;
const PURGE_GATE_KEY = 'kvShare:purge:gate';
const PURGE_GATE_SECONDS = 24 * 60 * 60;
// Rows per delete, with a pause between, so replicas keep up.
const PURGE_BATCH = 500;
const PURGE_PAUSE_MS = 50;
@@ -534,17 +536,7 @@ export class KvShareService extends PuterService {
#jobCursor: string | undefined;
override onServerStart(): void {
this.services.jobs.register(PURGE_JOB, {
every: PURGE_EVERY_MS,
scope: 'fleet',
nearPrimary: true,
leaseMs: PURGE_LEASE_MS,
run: async ({ signal }) => {
const purged = await this.purgeRevokedHandles(signal);
if (purged)
console.log(`[kvShare] purged ${purged} retired handles`);
},
});
this.#armPurge();
this.#armJobs();
// A check at one level is answered by a grant at any level above it.
@@ -615,24 +607,51 @@ export class KvShareService extends PuterService {
// -- Retention purge ---------------------------------------------
/**
* Delete handles retired longer ago than the retention, in batches, until
* none are left or `signal` aborts. Returns the rows deleted. A missing row
* reads like a retired one everywhere a handle is resolved.
* Delete handles retired longer ago than the retention, in batches. Returns
* the rows deleted, or null when another node holds today's gate. A missing
* row reads like a retired one everywhere a handle is resolved.
*/
async purgeRevokedHandles(signal: AbortSignal): Promise<number> {
async purgeRevokedHandles(): Promise<number | null> {
const claimed = await this.clients.redis.set(
PURGE_GATE_KEY,
String(Date.now()),
'EX',
PURGE_GATE_SECONDS,
'NX',
);
if (claimed !== 'OK') return null;
const cutoff =
Math.floor(Date.now() / 1000) - KV_SHARE_REVOKED_RETENTION_SECONDS;
let purged = 0;
for (let i = 0; i < PURGE_MAX_BATCHES && !signal.aborted; i++) {
const batch = await this.stores.kvShareHandle.deleteRevokedBefore(
cutoff,
PURGE_BATCH,
);
purged += batch;
if (batch < PURGE_BATCH) break;
await new Promise((resolve) => setTimeout(resolve, PURGE_PAUSE_MS));
let finished = false;
try {
for (let i = 0; i < PURGE_MAX_BATCHES; i++) {
if (this.#stopping) return purged;
const batch =
await this.stores.kvShareHandle.deleteRevokedBefore(
cutoff,
PURGE_BATCH,
);
purged += batch;
if (batch < PURGE_BATCH) break;
await new Promise((resolve) =>
setTimeout(resolve, PURGE_PAUSE_MS),
);
}
finished = true;
return purged;
} finally {
// A run cut short frees the gate, so another node picks it up on
// its next check instead of tomorrow.
if (!finished) {
try {
await this.clients.redis.del(PURGE_GATE_KEY);
} catch {
// The gate expires on its own.
}
}
}
return purged;
}
/**
@@ -1028,6 +1047,27 @@ export class KvShareService extends PuterService {
this.#timers.push(kick, timer);
}
#armPurge(): void {
const run = async () => {
try {
const purged = await this.purgeRevokedHandles();
if (purged)
console.log(`[kvShare] purged ${purged} retired handles`);
} catch (err) {
console.warn('[kvShare] retired-handle purge failed', err);
}
};
// Jittered so a deploy does not have every node check at once.
const kick = setTimeout(
() => void run(),
PURGE_INITIAL_DELAY_MS * (0.5 + Math.random()),
);
kick.unref?.();
const timer = setInterval(() => void run(), PURGE_CHECK_MS);
timer.unref?.();
this.#timers.push(kick, timer);
}
/** Whether the level a mint asks for is available on this server. */
#assertAccessEnabled(access: KvShareAccess): void {
if (access === 'watch') {
@@ -1705,14 +1745,11 @@ export class KvShareService extends PuterService {
* The other party of a share, for a notification one party sends through
* its handle from the app whose data it is. Every check runs before the
* pair budget is spent. A handle that is unknown, retired, not the
* caller's, or seen from another app reads as absent. `spendBudget: false`
* checks without spending, for a send that is only being scheduled. Called
* by extensions.
* caller's, or seen from another app reads as absent.
*/
async resolveNotifyRecipient(
actor: Actor,
handle: unknown,
{ spendBudget = true }: { spendBudget?: boolean } = {},
): Promise<KvShareNotifyTarget> {
const named = typeof handle === 'string' ? handle : '';
const senderId = actor.user?.id;
@@ -1745,8 +1782,7 @@ export class KvShareService extends PuterService {
if (!checked?.holds) throw unknownKvShareHandle(named);
if (checked.refuses) throw recipientNotAcceptingShares();
if (spendBudget)
await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
await this.#spendNotifyBudget(senderId, recipientId, share.appUid);
return {
recipientUserId: recipientId,
appUid: share.appUid,
@@ -111,11 +111,6 @@ type UsageWriteResult = {
detail?: FlatUsageDetail;
};
/** Compares fleet-wide spend against `maxGlobalUsagePerMinute`. */
const GLOBAL_RATE_CHECK_JOB = 'metering.globalRateCheck';
const GLOBAL_RATE_CHECK_EVERY_MS = 25 * 60 * 1000;
const GLOBAL_RATE_CHECK_LEASE_MS = 5 * 60 * 1000;
// -- Helpers ----------------------------------------------------------
/**
@@ -289,6 +284,7 @@ export class MeteringService extends PuterService {
*/
static APP_TOTALS_LIST_LIMIT = 4_000;
private rateCheckTimer: ReturnType<typeof setInterval> | null = null;
private usageBufferTimer: ReturnType<typeof setInterval> | null = null;
private extraPolicies: SubscriptionPolicy[] = [];
private subscriptionResolvers: SubscriptionResolver[] = [];
@@ -419,15 +415,15 @@ export class MeteringService extends PuterService {
},
);
const maxPerMinute = this.config.maxGlobalUsagePerMinute;
if (maxPerMinute && maxPerMinute > 0) {
this.services.jobs.register(GLOBAL_RATE_CHECK_JOB, {
every: GLOBAL_RATE_CHECK_EVERY_MS,
scope: 'fleet',
leaseMs: GLOBAL_RATE_CHECK_LEASE_MS,
run: () => this.checkRateOfChange(),
});
}
this.rateCheckTimer = setInterval(
() => {
this.checkRateOfChange().catch((e) => {
console.error('[metering] rate-of-change check failed', e);
});
},
1000 * 60 * 25,
);
this.rateCheckTimer.unref?.();
const flushInterval =
this.config.meteringUsageBufferFlushMs &&
@@ -458,6 +454,10 @@ export class MeteringService extends PuterService {
}
override async onServerShutdown(): Promise<void> {
if (this.rateCheckTimer) {
clearInterval(this.rateCheckTimer);
this.rateCheckTimer = null;
}
if (this.usageBufferTimer) {
clearInterval(this.usageBufferTimer);
this.usageBufferTimer = null;
@@ -2948,8 +2948,6 @@ export class MeteringService extends PuterService {
timestamp: number;
} | null;
// Several schedulers can share the baseline (one per region without a
// fleet scheduler, or old nodes mid-rollout); keep windows wide.
if (lastChange && now - lastChange.timestamp <= 14 * 60 * 1000) return;
const globalUsage = await this.getGlobalUsage();
@@ -395,36 +395,6 @@ describe('NotificationService.notifyUpdate', () => {
written.stop();
});
it('sends the desktop rewrite before reading the row back, so a failed read still updates it', async () => {
const user = await makeUser();
const uid = await notifications.notify(
[user.id],
{ title: 'one share' },
{ type: 'share.received' },
);
const pushed = collect('outer.gui.notif.message');
const read = vi
.spyOn(server.stores.notification, 'getByUid')
.mockRejectedValueOnce(new Error('replica down'));
try {
await expect(
notifications.notifyUpdate(
uid,
user.id,
{ title: 'two shares' },
{ type: 'share.received' },
),
).rejects.toThrow('replica down');
expect(pushed.seen).toHaveLength(1);
expect((pushed.seen[0] as NotifPush).response.notification).toEqual(
{ title: 'two shares', type: 'share.received' },
);
} finally {
read.mockRestore();
pushed.stop();
}
});
it('refuses to fold a type that does not group', async () => {
const user = await makeUser();
await expect(
@@ -912,42 +882,17 @@ describe('NotificationService.sweepExpired', () => {
});
it('stops at the pass cap and leaves the rest for the next sweep', async () => {
// A backlog that always fills the batch: the cap, not a short batch,
// is what ends the call. Seeding 500k rows would take minutes.
const deletes = vi
.spyOn(server.stores.notification, 'deleteCreatedBefore')
.mockResolvedValue(500);
try {
expect(await notifications.sweepExpired()).toBe(500_000);
expect(deletes).toHaveBeenCalledTimes(1000);
} finally {
deletes.mockRestore();
}
});
it('stops between batches once its signal aborts', async () => {
await notifications.sweepExpired();
const user = await makeUser();
await seedExpired(user.id, 1200);
const controller = new AbortController();
const deleteBatch = server.stores.notification.deleteCreatedBefore.bind(
server.stores.notification,
);
const spy = vi
.spyOn(server.stores.notification, 'deleteCreatedBefore')
.mockImplementation(async (days: number, limit: number) => {
const removed = await deleteBatch(days, limit);
controller.abort();
return removed;
});
try {
expect(await notifications.sweepExpired(controller.signal)).toBe(
500,
);
} finally {
spy.mockRestore();
}
expect(await notifications.sweepExpired()).toBe(700);
// 50 passes * 500/batch = 25,000 — a backlog past that so the cap,
// not a short batch, is what ends the first call.
await seedExpired(user.id, 25_050);
expect(await notifications.sweepExpired()).toBe(25_000);
expect(
await server.stores.notification.listByUserId(user.id),
).toHaveLength(50);
expect(await notifications.sweepExpired()).toBe(50);
});
it('sweeps nothing when no retention is configured', async () => {
@@ -48,16 +48,12 @@ export type {
} from './notificationTypes.js';
export { canViewNotification } from './notificationAudience.js';
const RETENTION_SWEEP_JOB = 'notifications.retention';
const RETENTION_SWEEP_EVERY_MS = 60 * 60 * 1000;
const RETENTION_SWEEP_LEASE_MS = 15 * 60 * 1000;
/** How often the retention sweep runs. */
const RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
/** Rows one delete takes. Small enough not to hold a lock anyone waits on. */
const RETENTION_BATCH_SIZE = 500;
/**
* Batches one sweep takes at most. It runs once for the fleet, so this is the
* fleet's hourly drain; the job's signal stops it sooner.
*/
const RETENTION_MAX_BATCHES = 1000;
/** Batches one sweep takes, so a large backlog drains over several passes. */
const RETENTION_MAX_BATCHES = 50;
/** Unread notifications one reconnect replays. */
const UNREAD_REPLAY_LIMIT = 200;
@@ -73,20 +69,11 @@ const UNREAD_REPLAY_LIMIT = 200;
export class NotificationService extends PuterService {
/** User.id → debounce timeout */
#connectTimeouts = new Map<number, ReturnType<typeof setTimeout>>();
#retentionSweep: ReturnType<typeof setInterval> | null = null;
#adapter: NotificationSocketAdapter | null = null;
override onServerStart(): void {
if (this.#retentionDays() > 0) {
this.services.jobs.register(RETENTION_SWEEP_JOB, {
every: RETENTION_SWEEP_EVERY_MS,
scope: 'fleet',
nearPrimary: true,
leaseMs: RETENTION_SWEEP_LEASE_MS,
run: async ({ signal }) => {
await this.sweepExpired(signal);
},
});
}
this.#armRetentionSweep();
this.#socket().attach();
// When a user opens the GUI, send their pending unreads.
@@ -116,6 +103,11 @@ export class NotificationService extends PuterService {
);
}
override onServerPrepareShutdown(): void {
if (this.#retentionSweep) clearInterval(this.#retentionSweep);
this.#retentionSweep = null;
}
// -- Public API --------------------------------------------------
/**
@@ -184,17 +176,25 @@ export class NotificationService extends PuterService {
}
if (!opts.silent) {
await this.emitWritten(
// Listeners get their own copy; the socket sends `payload`.
this.clients.event.emit(
'notif.written',
{
userId,
uid,
type: registered.type,
audience: registered.audience,
appUid,
value: payload,
value: structuredClone(payload),
sendId: uuidv4(),
},
{ sendId: uuidv4() },
{},
);
await this.#push(userId, uid, payload, {
type: registered.type,
audience: registered.audience,
appUid,
});
}
return uid;
}),
@@ -241,107 +241,50 @@ export class NotificationService extends PuterService {
if (!updated) return false;
if (opts.silent) return true;
// Before the read, so the desktop copy updates even if it fails.
const desktop = inAccountMailbox(registered.type);
if (desktop) this.#sendRewrite(userId, uid, payload);
if (desktop) {
// Same uid as the original: the client replaces what it is already
// showing rather than stacking another copy. It goes straight to
// the wire rather than through dispatch — the row was already
// published, and republishing it under its own id is a duplicate
// to anything deduplicating on `event.id`.
this.#socket().message(userId, uid, payload);
this.#markDelivered(uid, userId);
}
const row = await this.stores.notification.getByUid(uid, { userId });
const app = notification.app as { uid?: unknown } | undefined;
const appUid =
row?.app_uid ?? (typeof app?.uid === 'string' ? app.uid : null);
await this.emitWritten(
this.clients.event.emit(
'notif.written',
{
userId,
uid,
type: registered.type,
audience: registered.audience,
appUid,
value: payload,
updated: true,
replacedAt,
},
{ sendId: uuidv4(), rewriteSent: desktop },
);
return true;
}
/**
* Deliver a notification already written: `notif.written` for listeners,
* then the live copy. `sendId` names this one send; a delayed send passes
* the same one each time it delivers, so a listener can tell a repeat. An
* `updated` row is an earlier notification rewritten in place, and
* `replacedAt` marks it a replacement. `handlerDepth` is the depth the
* write was made at, when no actor in context carries it. `rewriteSent`
* says the caller already sent a desktop rewrite. `replayable` marks a send
* that may be emitted again under the same `sendId`. Called by extensions.
*/
async emitWritten(
written: {
userId: number;
uid: string;
type: string;
audience: string;
appUid: string | null;
value: Record<string, unknown>;
updated?: boolean;
replacedAt?: number;
},
opts: {
sendId: string;
handlerDepth?: number;
rewriteSent?: boolean;
replayable?: boolean;
},
): Promise<void> {
const { userId, uid, value: payload, updated } = written;
const scope = {
type: written.type,
audience: written.audience,
appUid: written.appUid,
};
const desktop = inAccountMailbox(scope.type);
if (updated && desktop && !opts.rewriteSent)
this.#sendRewrite(userId, uid, payload);
// Listeners get their own copy; the socket sends `payload`.
this.clients.event.emit(
'notif.written',
{
userId,
uid,
...scope,
value: structuredClone(payload),
...(updated ? { updated: true as const } : {}),
sendId: opts.sendId,
...(opts.replayable ? { replayable: true as const } : {}),
updated: true,
sendId: uuidv4(),
},
{},
);
// A rewrite of a type kept off the desktop has dispatch as its only
// live path, so it goes out there again, under the same uid.
if (!updated || !desktop)
// A type kept off the desktop has dispatch as its only live path, so
// the rewrite goes out there again, under the same uid.
if (!desktop) {
await this.#push(
userId,
uid,
payload,
scope,
written.replacedAt,
opts.handlerDepth,
{
type: registered.type,
audience: registered.audience,
appUid,
},
replacedAt,
);
}
/**
* Same uid as the original: the client replaces what it is already showing
* rather than stacking another copy. It goes straight to the wire rather
* than through dispatch — the row was already published, and republishing
* it under its own id is a duplicate to anything deduplicating on
* `event.id`.
*/
#sendRewrite(
userId: number,
uid: string,
payload: Record<string, unknown>,
): void {
this.#socket().message(userId, uid, payload);
this.#markDelivered(uid, userId);
}
return true;
}
/**
@@ -405,15 +348,17 @@ export class NotificationService extends PuterService {
* Drop notifications past the retention window, in batches, and report how
* many went. Deleting is all there is to it: nothing is pushed, because a
* two-week-old row is not news, and a client listing again simply stops
* seeing it. Stops between batches once `signal` aborts.
* seeing it.
*
* Every node sweeps. Batches are small and the delete is idempotent, so two
* nodes overlapping costs a few empty batches, not correctness.
*/
async sweepExpired(signal?: AbortSignal): Promise<number> {
async sweepExpired(): Promise<number> {
const days = this.#retentionDays();
if (days <= 0) return 0;
let removed = 0;
for (let pass = 0; pass < RETENTION_MAX_BATCHES; pass++) {
if (signal?.aborted) break;
const batch = await this.stores.notification.deleteCreatedBefore(
days,
RETENTION_BATCH_SIZE,
@@ -466,7 +411,6 @@ export class NotificationService extends PuterService {
payload: Record<string, unknown>,
scope: { type: string; audience: string; appUid: string | null },
replacedAt?: number,
depth?: number,
): Promise<void> {
const desktop = inAccountMailbox(scope.type);
if (!notificationsFoldInEnabled(this.config)) {
@@ -486,8 +430,9 @@ export class NotificationService extends PuterService {
return;
}
const handlerDepth =
depth ?? handlerDepthOf(Context.get('actor') as Actor | undefined);
const handlerDepth = handlerDepthOf(
Context.get('actor') as Actor | undefined,
);
this.clients.event.emit(
'notif.created',
{
@@ -525,6 +470,17 @@ export class NotificationService extends PuterService {
return Number.isFinite(configured) && configured > 0 ? configured : 0;
}
#armRetentionSweep(): void {
if (this.#retentionDays() <= 0) return;
const sweep = setInterval(() => {
void this.sweepExpired().catch((err) => {
console.warn('[notification] retention sweep failed', err);
});
}, RETENTION_SWEEP_INTERVAL_MS);
sweep.unref?.();
this.#retentionSweep = sweep;
}
/**
* What a reconnecting client missed. One statement marks the batch shown —
* a per-row update is a round trip per notification on a path that runs
@@ -243,61 +243,4 @@ describe('appMessageSender', () => {
'a'.repeat(120),
);
});
describe('an app made for an origin', () => {
const uid = 'app-65d98228-cfa5-5d18-a442-f6eeaf3dcfc7';
const originApp = (fields: Record<string, unknown> = {}) => ({
uid,
name: uid,
title: uid,
description: 'App created from origin https://chat.example.com',
index_url: 'https://chat.example.com',
...fields,
});
it('is named by the hostname of its index_url, without a port', () => {
expect(appMessageSender(uid, originApp()).name).toBe(
'chat.example.com',
);
expect(
appMessageSender(
uid,
originApp({
index_url: 'https://chat.example.com:8443/app/',
}),
).name,
).toBe('chat.example.com');
});
it('keeps its stored name when its index_url does not parse', () => {
expect(
appMessageSender(uid, originApp({ index_url: 'not a url' }))
.name,
).toBe(uid);
expect(
appMessageSender(uid, originApp({ index_url: undefined })).name,
).toBe(uid);
});
it('keeps a title its developer set', () => {
expect(
appMessageSender(uid, originApp({ title: 'Chat' })).name,
).toBe('Chat');
});
it('leaves apps that were not made for an origin alone', () => {
expect(
appMessageSender(uid, originApp({ description: 'My chat app' }))
.name,
).toBe(uid);
expect(
appMessageSender('app-1', {
uid: 'app-1',
name: 'chat',
title: 'Chat',
index_url: 'https://chat.example.com',
}).name,
).toBe('Chat');
});
});
});
@@ -18,7 +18,6 @@
*/
import { HttpError } from '../../core/http/HttpError.js';
import { isOriginBootstrapApp } from '../../util/appView.js';
/** The one notification type an app may create. */
export const APP_MESSAGE_TYPE = 'app.message';
@@ -61,35 +60,14 @@ export type AppMessage = {
export type AppMessageSender = { uid: string; name?: string; icon?: string };
/** The hostname an app made for an origin is named by. */
const originAppHost = (row: unknown): string | undefined => {
if (!isOriginBootstrapApp(row) || typeof row.index_url !== 'string') {
return undefined;
}
try {
return new URL(row.index_url).hostname || undefined;
} catch {
return undefined;
}
};
/** What a message says about its app, read off the app's row. */
export const appMessageSender = (
uid: string,
row:
| {
name?: unknown;
title?: unknown;
icon?: unknown;
description?: unknown;
index_url?: unknown;
}
| null
| undefined,
row: { name?: unknown; title?: unknown; icon?: unknown } | null | undefined,
): AppMessageSender => {
const sender: AppMessageSender = { uid };
// Cleaned before the empty check, so a title of only invisible characters still leaves a visible sender.
const name = [originAppHost(row), row?.title, row?.name]
const name = [row?.title, row?.name]
.map((v) =>
typeof v === 'string'
? [...cleanText(v)]
@@ -3,19 +3,18 @@
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see
* [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { describe, expect, it } from 'vitest';
@@ -54,9 +53,6 @@ describe('NOTIFICATION_TYPES', () => {
audience: 'developer',
appScoped: true,
});
expect(
findNotificationType('app.events.handler_suspended'),
).toMatchObject({ audience: 'app-user', appScoped: true });
expect(findNotificationType('app.events.ended')).toMatchObject({
audience: 'app-user',
appScoped: false,
@@ -65,9 +61,9 @@ describe('NOTIFICATION_TYPES', () => {
it('projects a subject naming the app, or the recipient when there is none', () => {
const account = findNotificationType('share.received')!;
expect(account.subject({ userUuid: 'user-uuid', appUid: null })).toBe(
'notif:user-uuid:account',
);
expect(
account.subject({ userUuid: 'user-uuid', appUid: null }),
).toBe('notif:user-uuid:account');
const developer = findNotificationType('app.events.suspended')!;
expect(
@@ -134,20 +130,18 @@ describe('resolveNotificationWrite', () => {
});
it('requires an app uid for an app message', () => {
expect(resolveNotificationWrite('app.message', 'app-1')).toMatchObject({
audience: 'app-user',
appScoped: true,
groupable: true,
});
expect(resolveNotificationWrite('app.message', 'app-1')).toMatchObject(
{ audience: 'app-user', appScoped: true, groupable: true },
);
expect(() => resolveNotificationWrite('app.message', null)).toThrow(
'requires an app uid',
);
});
it('lets a subscription an account made end without naming an app', () => {
expect(
resolveNotificationWrite('app.events.ended', null).audience,
).toBe('app-user');
expect(resolveNotificationWrite('app.events.ended', null).audience).toBe(
'app-user',
);
});
});
@@ -115,15 +115,6 @@ export const NOTIFICATION_TYPES = [
groupable: false,
subject: subjectFor('developer'),
},
// A handler the user published into their own scope stopped taking
// deliveries: the code is theirs, so the notice is too.
{
type: 'app.events.handler_suspended',
audience: 'app-user',
appScoped: true,
groupable: false,
subject: subjectFor('app-user'),
},
// The only type apps create; its kind goes in the value's `channel`. A
// newer one replaces the newest with the same `group` unless that one was
// shown or dismissed.
-3
View File
@@ -17,7 +17,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type { ScheduleServiceContract } from '../core/schedules.js';
import type { TeamServiceContract } from '../core/teams.js';
import type { puterClients } from '../clients';
@@ -39,8 +38,6 @@ export interface IPuterServiceInstances {}
export interface IExtensionServiceInstances {
/** Teams ships separately; boot guarantees it when `teams_enabled` is on. */
team?: TeamServiceContract;
/** Schedules ship separately; without them their routes answer 503. */
schedules?: ScheduleServiceContract;
[key: string]: unknown;
}
@@ -381,29 +381,3 @@ describe('SignupService.confirmEmail', () => {
expect(after!.email).toBe(other.email);
});
});
describe('SignupService.validateEmail', () => {
it('refuses Puter mailbox addresses before any listener runs', async () => {
const listener = vi.fn();
server.clients.event.on('email.validate', listener);
try {
for (const email of [
'[email protected]',
'[email protected]',
'[email protected]',
]) {
await expect(
server.services.signup.validateEmail(email),
).rejects.toMatchObject({ statusCode: 400 });
}
expect(listener).not.toHaveBeenCalled();
await expect(
server.services.signup.validateEmail(
`ok-${slug()}@example.com`,
),
).resolves.toBeUndefined();
} finally {
server.clients.event.off('email.validate', listener);
}
});
});
+2 -13
View File
@@ -26,7 +26,6 @@ import { HttpError } from '../../core/http/HttpError.js';
import type { UserRow } from '../../stores/user/UserStore.js';
import { isOwnedEmailConflict } from '../../stores/user/UserStore.js';
import { cleanEmail, isBlockedEmail } from '../../util/email.js';
import { isPuterEmailAddress } from '../email/mailbox.js';
import {
bonusCodeInvalidError,
checkSignupBonus,
@@ -96,20 +95,10 @@ export const signupClientIp = (req: Request): string | null =>
*/
export class SignupService extends PuterService {
/**
* Puter mailbox addresses, the config blocklist (suffix match on the
* cleaned address), then the `email.validate` hook. Throws 400 on
* rejection.
* Config blocklist (suffix match on the cleaned address), then the
* `email.validate` hook. Throws 400 on rejection.
*/
async validateEmail(email: string): Promise<void> {
// A Puter mailbox can't be an account's own email: it is reached
// through an account in the first place.
if (isPuterEmailAddress(email.trim())) {
throw new HttpError(
400,
'A Puter email address cannot be used as your account email. Please use a different email address.',
{ legacyCode: 'email_not_allowed' as never },
);
}
if (isBlockedEmail(email, this.config.blockedEmailDomains)) {
throw new HttpError(400, 'This email is not allowed.', {
legacyCode: 'email_not_allowed' as never,
+1 -14
View File
@@ -262,19 +262,6 @@ export class AppStore extends PuterStore {
return result;
}
/**
* Cache-free existence check by uid against the primary. Called by
* extensions.
*/
async uidExistsOnPrimary(uid) {
if (!uid) return false;
const rows = await this.clients.db.pread(
'SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1',
[uid],
);
return rows.length > 0;
}
/** Cache-free existence check against the primary. */
async existsOnPrimary(appId) {
if (appId === undefined || appId === null) return false;
@@ -801,7 +788,7 @@ export class AppStore extends PuterStore {
* the row stays unclaimed until a developer registers an app for the same
* origin and AppDriver's merge path takes ownership. Marker shape (`name
* === uid && title === uid` + description prefix) is what
* `isOriginBootstrapApp` (util/appView) checks.
* `#isOriginBootstrapApp` checks.
*/
async createFromOrigin(
uid,
@@ -43,7 +43,6 @@ import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import { EVENT_SUBSCRIPTION_WIDTHS } from './columnWidths.js';
import type { DurableSubscriptionInput } from './DurableSubscriptionStore.js';
import { scheduleToken } from './types.js';
const BOOT_TIMEOUT_MS = 120_000;
@@ -202,9 +201,9 @@ describe('creating a subscription', () => {
const { row } = await durable().create(input());
await expect(cache().userHasAny(userId)).resolves.toBe(true);
await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
[token()],
);
await expect(
cache().watchedTokens(userId, [token()]),
).resolves.toEqual([token()]);
await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
row,
]);
@@ -265,114 +264,6 @@ describe('creating a subscription', () => {
});
});
describe('handler scope', () => {
const appUid = 'app-scoped';
it('round-trips through create and every select path', async () => {
const { row: user } = await durable().create(
input({ appUid, handlerName: 'h', handlerScope: 'user' }),
);
const { row: app } = await durable().create(
input({ appUid, handlerName: 'h' }),
);
expect(user.handlerScope).toBe('user');
expect(app).not.toHaveProperty('handlerScope');
await expect(durable().getBySubId(user.subId)).resolves.toMatchObject({
handlerScope: 'user',
});
await expect(
durable().getBySubId(app.subId),
).resolves.not.toHaveProperty('handlerScope');
const listed = await durable().listForHolder(userId, { appUid });
expect(
new Map(listed.items.map((r) => [r.subId, r.handlerScope])),
).toEqual(
new Map<string, string | undefined>([
[user.subId, 'user'],
[app.subId, undefined],
]),
);
const stored = await env.server.clients.db.read(
'SELECT `sub_id`, `handler_scope` FROM `event_subscriptions` WHERE `app_uid` = ?',
[appUid],
);
expect(
Object.fromEntries(stored.map((r) => [r.sub_id, r.handler_scope])),
).toEqual({ [user.subId]: 1, [app.subId]: 0 });
});
it('lists by scope and holder, never across scopes', async () => {
const { row: appRow } = await durable().create(
input({ appUid, handlerName: 'h' }),
);
const { row: mine } = await durable().create(
input({ appUid, handlerName: 'h', handlerScope: 'user' }),
);
const { row: theirs } = await durable().create(
input({
appUid,
handlerName: 'h',
handlerScope: 'user',
holderUserId: otherUserId,
ownerUserId: otherUserId,
token: `f#other-${uuidv4()}`,
}),
);
const ids = async (scope: 'app' | 'user', holder: number, name = 'h') =>
(await durable().listByHandler(appUid, scope, holder, name))
.map((r) => r.subId)
.sort();
expect(await ids('app', 0)).toEqual([appRow.subId]);
// Holder is ignored for app scope.
expect(await ids('app', userId)).toEqual([appRow.subId]);
expect(await ids('user', userId)).toEqual([mine.subId]);
expect(await ids('user', otherUserId)).toEqual([theirs.subId]);
expect(await ids('user', userId, 'missing')).toEqual([]);
});
it('lists the suspended half of a user scope', async () => {
const { row } = await durable().create(
input({ appUid, handlerName: 'h', handlerScope: 'user' }),
);
await env.server.clients.db.write(
'UPDATE `event_subscriptions` SET `suspended_at` = 1, `suspended_reason` = ? WHERE `sub_id` = ?',
['handler_not_found', row.subId],
);
await expect(
durable().listByHandler(appUid, 'user', userId, 'h'),
).resolves.toEqual([]);
await expect(
durable().listByHandler(appUid, 'user', userId, 'h', {
suspendedReason: 'handler_not_found',
}),
).resolves.toEqual([expect.objectContaining({ subId: row.subId })]);
});
it('reads a user scope through the holder index', async () => {
const pread = vi.spyOn(env.server.clients.db, 'pread');
try {
await durable().listByHandler(appUid, 'user', userId, 'h');
const [statement, params] = pread.mock.calls[0]!;
const plan = await env.server.clients.db.read(
`EXPLAIN QUERY PLAN ${statement}`,
params,
);
const detail = plan.map((r) => String(r.detail)).join('\n');
expect(detail).toContain('idx_event_subscriptions_holder');
expect(detail).not.toContain('idx_event_subscriptions_app_handler');
} finally {
pread.mockRestore();
}
});
it('carries the scope through the schedule join', async () => {
const join = durable().scheduleSubscriptionJoin('s', 'x.`sub_id`');
expect(join.columns).toContain('`sub_handler_scope`');
});
});
describe('validation at the row write', () => {
it('refuses a context past the hard cap', async () => {
await expect(
@@ -577,9 +468,7 @@ describe('the per-account cap', () => {
input({ expiresAt: Math.floor(Date.now() / 1000) - 60 }),
);
await durable().create(input());
await expect(durable().countForHolder(userId)).resolves.toMatchObject({
total: 1,
});
await expect(durable().countForHolder(userId)).resolves.toMatchObject({ total: 1 });
});
});
@@ -658,13 +547,9 @@ describe('rows suspended for good', () => {
await durable().suspend([failing], 'failures');
await durable().suspend([revoked], 'permission_revoked');
const { rows: held } = await durable().listActiveForHolderPage(
userId,
null,
0,
undefined,
{ includeSuspended: true },
);
const held = await durable().listActiveForHolder(userId, null, {
includeSuspended: true,
});
expect(held.map((row) => row.subId).sort()).toEqual(
[live.subId, failing.subId].sort(),
);
@@ -680,9 +565,7 @@ describe('rows suspended for good', () => {
},
);
const created = await durable().create(input());
// The trim's bump, for the caller to publish to the other regions.
expect(created.reaped.map((bump) => bump.userId)).toEqual([userId]);
await durable().create(input());
await expect(durable().countForHolder(userId)).resolves.toEqual({
total: 1,
@@ -717,9 +600,9 @@ describe('removal', () => {
await durable().remove(row);
await expect(cache().watchedTokens(userId, [token()])).resolves.toEqual(
[token()],
);
await expect(
cache().watchedTokens(userId, [token()]),
).resolves.toEqual([token()]);
await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
survivor.row,
]);
@@ -861,11 +744,9 @@ describe('the expiry sweep', () => {
// one back to prove the sweep is what removes it.
await cache().cacheDurable([expired.row]);
expect((await durable().sweepExpired(500)).rows).toHaveLength(1);
await expect(durable().sweepExpired(500)).resolves.toBe(1);
await expect(
durable().getBySubId(expired.row.subId),
).resolves.toBeNull();
await expect(durable().getBySubId(expired.row.subId)).resolves.toBeNull();
await expect(cache().getForTokens(userId, [token()])).resolves.toEqual([
live.row,
]);
@@ -873,42 +754,19 @@ describe('the expiry sweep', () => {
it('leaves a subscription with no expiry alone', async () => {
await durable().create(input());
expect((await durable().sweepExpired(500)).rows).toHaveLength(0);
await expect(durable().sweepExpired(500)).resolves.toBe(0);
await expect(durable().countForHolder(userId)).resolves.toMatchObject({
total: 1,
});
});
});
describe('a sweep reading a lagging replica', () => {
it('keeps a row the primary resumed since, and does not report it', async () => {
const { row } = await durable().create(input());
await durable().suspend([row], 'permission_revoked');
const [stale] = await env.server.clients.db.pread(
'SELECT * FROM `event_subscriptions` WHERE `sub_id` = ?',
[row.subId],
);
await durable().resume([row]);
// The replica still has it suspended.
vi.spyOn(env.server.clients.db, 'read').mockResolvedValueOnce([stale]);
const reaped = await durable().sweepSuspended(
Math.floor(Date.now() / 1000) + 3600,
500,
);
expect(reaped.rows).toEqual([]);
await expect(durable().getBySubId(row.subId)).resolves.toMatchObject({
subId: row.subId,
suspendedAt: null,
});
});
});
describe('listHolderIdsForApp', () => {
it('names every distinct holder bound to the app`s handlers', async () => {
const appUid = `app-${uuidv4()}`;
await durable().create(input({ appUid, holderUserId: userId }));
await durable().create(
input({ appUid, holderUserId: userId }),
);
await durable().create(
input({
appUid,
@@ -916,7 +774,9 @@ describe('listHolderIdsForApp', () => {
}),
);
// A second row for the same holder must not double the name.
await durable().create(input({ appUid, holderUserId: userId }));
await durable().create(
input({ appUid, holderUserId: userId }),
);
const holders = await durable().listHolderIdsForApp(appUid);
expect(holders.sort()).toEqual([userId, otherUserId].sort());
@@ -924,44 +784,48 @@ describe('listHolderIdsForApp', () => {
it('finds a suspended row`s holder too', async () => {
const appUid = `app-${uuidv4()}`;
const { row } = await durable().create(input({ appUid }));
const { row } = await durable().create(
input({ appUid }),
);
await durable().suspend([row], 'handler_not_found');
expect(await durable().listHolderIdsForApp(appUid)).toEqual([userId]);
});
it('answers an empty list for an app with nothing bound', async () => {
expect(await durable().listHolderIdsForApp(`app-${uuidv4()}`)).toEqual(
[],
);
expect(
await durable().listHolderIdsForApp(`app-${uuidv4()}`),
).toEqual([]);
});
});
describe('reapForApp', () => {
it('deletes every row bound to the app and bumps its owner`s generation', async () => {
const appUid = `app-${uuidv4()}`;
const mine = await durable().create(input({ appUid }));
const mine = await durable().create(
input({ appUid }),
);
const untouched = await durable().create(input());
const before = await cache().getGeneration(userId);
const reaped = await durable().reapForApp(appUid, 500);
expect(reaped.rows.map((row) => row.subId)).toEqual([mine.row.subId]);
await expect(durable().reapForApp(appUid, 500)).resolves.toBe(1);
await expect(durable().getBySubId(mine.row.subId)).resolves.toBeNull();
await expect(
durable().getBySubId(untouched.row.subId),
).resolves.not.toBeNull();
const after = await cache().getGeneration(userId);
expect(after).toBeGreaterThan(before);
// The bump the caller publishes to the other regions.
expect(reaped.bumps).toEqual([{ userId, generation: after }]);
await expect(cache().getGeneration(userId)).resolves.toBeGreaterThan(
before,
);
});
it('never touches another app`s rows', async () => {
const appUid = `app-${uuidv4()}`;
const otherAppUid = `app-${uuidv4()}`;
await durable().create(input({ appUid }));
const theirs = await durable().create(input({ appUid: otherAppUid }));
const theirs = await durable().create(
input({ appUid: otherAppUid }),
);
await durable().reapForApp(appUid, 500);
@@ -973,7 +837,7 @@ describe('reapForApp', () => {
it('answers 0 for an app with nothing to reap', async () => {
await expect(
durable().reapForApp(`app-${uuidv4()}`, 500),
).resolves.toEqual({ rows: [], bumps: [] });
).resolves.toBe(0);
});
});
@@ -1040,298 +904,3 @@ describe('suspending', () => {
expect(second.bumps).toEqual([]);
});
});
describe('rows a schedule owns', () => {
const scheduleInput = (
over: Partial<DurableSubscriptionInput> = {},
): DurableSubscriptionInput => {
const uid = uuidv4();
return input({
subject: scheduleToken(uid),
token: scheduleToken(uid),
anchorUid: uid,
anchorPath: '',
appUid: 'app-scheduler',
delivery: 'single',
handlerName: 'tick',
permission: 'events:background',
...over,
});
};
it('are created past the durable quota, uncached and with no bump', async () => {
const limits = { perUser: 1, perApp: 1 };
await durable().create(input({ limits }));
const before = await cache().getGeneration(userId);
const created = await durable().create(scheduleInput({ limits }));
expect(created.bump).toBeNull();
await expect(cache().getGeneration(userId)).resolves.toBe(before);
await expect(
cache().getForTokens(userId, [created.row.token]),
).resolves.toEqual([]);
await expect(
durable().getBySubId(created.row.subId),
).resolves.toMatchObject({ token: created.row.token });
});
it('never enter the dispatch cache or what a cold region rebuilds', async () => {
const { row } = await durable().create(scheduleInput());
await expect(
durable().listDeliverableForOwner(userId),
).resolves.toEqual([]);
await cache().markRegionCold(userId);
await durable().warmRegion(userId);
await expect(cache().userHasAny(userId)).resolves.toBe(false);
await durable().suspend([row], 'no_credit');
await durable().resume([
{ ...row, suspendedAt: 1, suspendedReason: 'no_credit' },
]);
await expect(
cache().getForTokens(userId, [row.token]),
).resolves.toEqual([]);
await expect(cache().userHasAny(userId)).resolves.toBe(false);
});
it('bump no generation when suspended or resumed', async () => {
const { row } = await durable().create(scheduleInput());
const before = await cache().getGeneration(userId);
const { suspended, bumps } = await durable().suspend([row], 'failures');
expect(suspended).toHaveLength(1);
expect(bumps).toEqual([]);
await expect(durable().resume(suspended)).resolves.toEqual([]);
await expect(cache().getGeneration(userId)).resolves.toBe(before);
});
it('count toward no durable quota, and are not in the durable listing', async () => {
await durable().create(input());
await durable().create(scheduleInput());
await expect(durable().countForHolder(userId)).resolves.toEqual({
total: 1,
forApp: 0,
terminal: 0,
});
const listed = await durable().listForHolder(userId, {
includeTotal: true,
});
expect(listed.total).toBe(1);
expect(listed.items.map((row) => row.token)).toEqual([token()]);
});
it('are kept when the oldest revoked rows are trimmed', async () => {
const base = Math.floor(Date.now() / 1000) - 10_000;
const { row: scheduled } = await durable().create(scheduleInput());
await env.server.clients.db.write(
'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
'`suspended_reason` = ? WHERE `sub_id` = ?',
[base - 1, 'permission_revoked', scheduled.subId],
);
for (let i = 0; i < EVENTS_REVOKED_SUBSCRIPTIONS_KEPT + 1; i++) {
const { row } = await durable().create(input());
await env.server.clients.db.write(
'UPDATE `event_subscriptions` SET `suspended_at` = ?, ' +
'`suspended_reason` = ? WHERE `sub_id` = ?',
[base + i, 'permission_revoked', row.subId],
);
}
await durable().create(input());
await expect(
durable().getBySubId(scheduled.subId),
).resolves.not.toBeNull();
await expect(durable().countForHolder(userId)).resolves.toMatchObject({
terminal: EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
});
});
it('are read, a page at a time, by what settles a holder', async () => {
const made: string[] = [];
for (let i = 0; i < 3; i++)
made.push((await durable().create(scheduleInput())).row.subId);
made.push((await durable().create(input())).row.subId);
const first = await durable().listActiveForHolderPage(
userId,
null,
0,
2,
);
expect(first.rows.map((row) => row.subId)).toEqual(made.slice(0, 2));
expect(first.nextId).not.toBeNull();
const second = await durable().listActiveForHolderPage(
userId,
null,
first.nextId!,
2,
);
expect(second.rows.map((row) => row.subId)).toEqual(made.slice(2));
// A full page always says to look again; the next one is empty.
const third = await durable().listActiveForHolderPage(
userId,
null,
second.nextId ?? Number.MAX_SAFE_INTEGER,
2,
);
expect(third).toEqual({ rows: [], nextId: null });
});
it('are read, a page at a time, by what a top-up resumes', async () => {
const rows = [];
for (let i = 0; i < 3; i++)
rows.push((await durable().create(scheduleInput())).row);
await durable().suspend(rows, 'no_credit');
const first = await durable().listSuspendedForHolderPage(
userId,
'no_credit',
0,
2,
);
expect(first.rows).toHaveLength(2);
const second = await durable().listSuspendedForHolderPage(
userId,
'no_credit',
first.nextId!,
2,
);
expect(second.rows.map((row) => row.subId)).toEqual([rows[2].subId]);
expect(second.nextId).toBeNull();
});
it('are found by handler and by app, like any other row', async () => {
const { row } = await durable().create(scheduleInput());
await expect(
durable().listByHandler('app-scheduler', 'app', 0, 'tick'),
).resolves.toEqual([expect.objectContaining({ subId: row.subId })]);
await expect(
durable().listHolderIdsForApp('app-scheduler'),
).resolves.toEqual([userId]);
});
it('come back from a reap without touching a backlog', async () => {
const { row } = await durable().create(scheduleInput());
const regular = await durable().create(
input({ appUid: 'app-scheduler' }),
);
const purge = vi.spyOn(env.server.stores.pendingDelivery, 'purge');
try {
const reaped = await durable().reapForApp('app-scheduler', 500);
expect(reaped.rows.map((r) => r.subId).sort()).toEqual(
[row.subId, regular.row.subId].sort(),
);
expect(purge.mock.calls.map(([subId]) => subId)).toEqual([
regular.row.subId,
]);
// Only the cached row's owner was bumped.
expect(reaped.bumps).toHaveLength(1);
} finally {
purge.mockRestore();
}
});
it('are deleted by their schedule`s id, whether or not it recorded the row', async () => {
const uid = uuidv4();
await durable().create(
scheduleInput({
subject: scheduleToken(uid),
token: scheduleToken(uid),
anchorUid: uid,
}),
);
await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
true,
);
await expect(durable().removeScheduleSubscription(uid)).resolves.toBe(
false,
);
});
});
describe('statements a schedule store composes', () => {
const createScheduleRow = async () => {
const uid = uuidv4();
const { row } = await durable().create(
input({
subject: scheduleToken(uid),
token: scheduleToken(uid),
anchorUid: uid,
anchorPath: '',
appUid: 'app-scheduler',
delivery: 'single',
handlerName: 'tick',
context: '{"region":"eu"}',
permission: 'events:background',
}),
);
return { uid, row };
};
it('delete a schedule`s row only while the guard holds', async () => {
const { uid, row } = await createScheduleRow();
const db = env.server.clients.db;
await db.batchWrite([
durable().scheduleSubscriptionDeleteEntry(uid, {
table: 'user',
where: '`id` = ?',
values: [-1],
}),
]);
await expect(durable().getBySubId(row.subId)).resolves.not.toBeNull();
await db.batchWrite([
durable().scheduleSubscriptionDeleteEntry(uid, {
table: 'user',
where: '`id` = ?',
values: [userId],
}),
]);
await expect(durable().getBySubId(row.subId)).resolves.toBeNull();
});
it('join a schedule`s row under the sub_* columns', async () => {
const { row } = await createScheduleRow();
const join = durable().scheduleSubscriptionJoin('sub', 'x.`sub_id`');
const read = (subId: string) =>
env.server.clients.db.pread(
`SELECT ${join.columns} FROM (SELECT ? AS \`sub_id\`) x ${join.join}`,
[subId],
);
const [found] = await read(row.subId);
expect(found).toMatchObject({
sub_suspended_at: null,
sub_suspended_reason: null,
sub_handler_name: 'tick',
});
expect(Number(found.sub_row_id)).toBeGreaterThan(0);
expect(found.sub_context).toBe('{"region":"eu"}');
const [missing] = await read('no-such-sub');
expect(missing).toMatchObject({
sub_row_id: null,
sub_handler_name: null,
});
});
it('tell a schedule whose row is gone', async () => {
const { uid, row } = await createScheduleRow();
const absent = async (subId: string) => {
const [result] = await env.server.clients.db.pread(
`SELECT CASE WHEN ${durable().scheduleSubscriptionAbsent('?')} THEN 1 ELSE 0 END AS \`absent\``,
[subId],
);
return Number(result.absent);
};
await expect(absent(row.subId)).resolves.toBe(0);
await durable().removeScheduleSubscription(uid);
await expect(absent(row.subId)).resolves.toBe(1);
});
});
@@ -23,9 +23,7 @@ import {
EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
type TieredQuota,
} from '../../controllers/events/limits.js';
import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
import { HttpError } from '../../core/http/HttpError.js';
import type { SqlGuard } from '../../core/schedules.js';
import type { DeliveryClass } from '../../services/events/registry.js';
import type { FsOp } from '../../services/events/subjects.js';
import {
@@ -39,14 +37,7 @@ import {
EVENT_SUBSCRIPTION_WIDTHS,
} from './columnWidths.js';
import type { GenerationBump } from './EventSubscriptionStore.js';
import {
isScheduleRow,
SCHEDULE_TOKEN_PREFIX,
scheduleToken,
type DurableSubscription,
type HandlerScopeKind,
type SubscriptionPermission,
} from './types.js';
import type { DurableSubscription, SubscriptionPermission } from './types.js';
/**
* Subscriptions that outlive the connection that made them.
@@ -75,9 +66,6 @@ export const DURABLE_CONTEXT_MAX_BYTES = 4096;
export const DURABLE_LIST_DEFAULT_LIMIT = 50;
export const DURABLE_LIST_LIMIT_CAP = 200;
/** Rows one page of a holder-wide settle takes. */
export const HOLDER_SETTLE_BATCH = 500;
/**
* Rows one handler-lifecycle pass takes. A removal suspends its dependents in
* batches so a widely-used name cannot make one call hold the whole set.
@@ -101,8 +89,6 @@ export interface DurableSubscriptionInput {
op: FsOp | null;
delivery: DeliveryClass;
handlerName: string | null;
/** Which set `handlerName` refers to. Omitted means the app's. */
handlerScope?: HandlerScopeKind;
context: string | null;
permission: SubscriptionPermission;
expiresAt: number | null;
@@ -115,15 +101,6 @@ export interface DurableSubscriptionInput {
limits?: TieredQuota;
}
/**
* What a reap deleted, and the generation it bumped per owner. The caller
* publishes the bumps, since the reap only invalidates this region.
*/
export interface DurableReap {
rows: DurableSubscription[];
bumps: GenerationBump[];
}
/** One keyset page of rows, and where the next page starts. */
export interface DurablePage {
rows: DurableSubscription[];
@@ -171,13 +148,6 @@ const TERMINAL_CLAUSE =
'(`suspended_at` IS NOT NULL AND `suspended_reason` IN (' +
`${TERMINAL_SUSPENDED_REASONS.map(() => '?').join(', ')}))`;
/**
* Leaves out the rows schedules own: they have their own caps and listing, and
* never enter the dispatch cache. Binds {@link SCHEDULE_TOKEN_PATTERN}.
*/
const NOT_SCHEDULE_CLAUSE = '`token` NOT LIKE ?';
const SCHEDULE_TOKEN_PATTERN = `${SCHEDULE_TOKEN_PREFIX}%`;
/** Where a row is moving to when its anchor is deleted under it. */
export interface ReanchorInput {
token: string;
@@ -197,12 +167,6 @@ const contextTooLarge = (): HttpError =>
{ legacyCode: 'events_context_too_large' },
);
export const assertDurableContext = (context: string | null): void => {
if (context === null) return;
if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
throw contextTooLarge();
};
const quotaReached = (limit: number, scope: 'account' | 'app'): HttpError =>
new HttpError(
429,
@@ -243,16 +207,6 @@ const quotaRefusal = (
return null;
};
/**
* A created row; `reaped` holds the bumps of terminal rows it trimmed. A
* schedule's row is never cached, so it comes back with no bump.
*/
export interface DurableCreated {
row: DurableSubscription;
bump: GenerationBump | null;
reaped: GenerationBump[];
}
/**
* The quota refusal for a create that concurrent creates beat to the last slot
* after its own insert. Its row is already gone again; `bump` still has to be
@@ -310,9 +264,6 @@ const toRow = (row: Record<string, unknown>): DurableSubscription => ({
row.handler_name === null || row.handler_name === undefined
? null
: String(row.handler_name),
...(Number(row.handler_scope) === 1
? { handlerScope: 'user' as const }
: {}),
context:
row.context === null || row.context === undefined
? null
@@ -330,7 +281,7 @@ const toRow = (row: Record<string, unknown>): DurableSubscription => ({
const SELECT_COLUMNS =
'`id`, `sub_id`, `token`, `owner_user_id`, `holder_user_id`, `app_uid`, ' +
'`subject`, `anchor_uid`, `anchor_path`, `match`, `delivery`, `ops`, ' +
'`handler_name`, `handler_scope`, `context`, `permission`, `include_value`, ' +
'`handler_name`, `context`, `permission`, `include_value`, ' +
'`suspended_at`, `suspended_reason`, `expires_at`, `created_at`';
export class DurableSubscriptionStore extends PuterStore {
@@ -340,13 +291,11 @@ export class DurableSubscriptionStore extends PuterStore {
* Insert one subscription and put it in this region's cache before
* returning. The generation bump is what tells every other region, and the
* caller broadcasts it.
*
* A schedule's row skips the durable quotas and the cache, so it comes back
* with no bump: its caps are the schedule store's.
*/
async create(input: DurableSubscriptionInput): Promise<DurableCreated> {
assertDurableContext(input.context);
const forSchedule = isScheduleRow(input);
async create(
input: DurableSubscriptionInput,
): Promise<{ row: DurableSubscription; bump: GenerationBump }> {
this.#assertContext(input.context);
assertColumnWidths(EVENT_SUBSCRIPTION_WIDTHS, {
token: input.token,
appUid: input.appUid,
@@ -358,14 +307,12 @@ export class DurableSubscriptionStore extends PuterStore {
permission: input.permission,
});
if (!forSchedule) {
const refusal = quotaRefusal(
input,
await this.countForHolder(input.holderUserId, input.appUid),
0,
);
if (refusal) throw refusal;
}
const refusal = quotaRefusal(
input,
await this.countForHolder(input.holderUserId, input.appUid),
0,
);
if (refusal) throw refusal;
const row: DurableSubscription = {
durable: true,
@@ -382,9 +329,6 @@ export class DurableSubscriptionStore extends PuterStore {
permission: input.permission,
delivery: input.delivery,
handlerName: input.handlerName,
...(input.handlerScope === 'user'
? { handlerScope: 'user' as const }
: {}),
context: input.context,
...(input.includeValue ? { includeValue: true } : {}),
expiresAt: input.expiresAt,
@@ -407,7 +351,6 @@ export class DurableSubscriptionStore extends PuterStore {
delivery: row.delivery,
ops: row.op,
handler_name: row.handlerName,
handler_scope: row.handlerScope === 'user' ? 1 : 0,
targets: legacyTargets(row.appUid),
context: row.context,
permission: row.permission,
@@ -415,7 +358,6 @@ export class DurableSubscriptionStore extends PuterStore {
expires_at: row.expiresAt,
created_at: row.createdAt,
});
if (forSchedule) return { row, bump: null, reaped: [] };
// Concurrent creates can all pass the count above. Counted again with
// this row in it, each one past the cap takes itself back out — at
@@ -426,20 +368,18 @@ export class DurableSubscriptionStore extends PuterStore {
);
const lost = quotaRefusal(input, held, 1);
if (lost) throw new DurableQuotaRaceLost(lost, await this.remove(row));
const reaped =
held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT
? await this.#dropOldestTerminal(
input.holderUserId,
held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
)
: [];
if (held.terminal > EVENTS_REVOKED_SUBSCRIPTIONS_KEPT)
await this.#dropOldestTerminal(
input.holderUserId,
held.terminal - EVENTS_REVOKED_SUBSCRIPTIONS_KEPT,
);
// Write-through, and over the whole owner rather than the one row: it
// costs the same indexed read as a warm would, on a path rate-limited
// to a few calls a minute, and it leaves this region needing nothing
// from the table before it can deliver.
await this.#rebuildRegion(row.ownerUserId);
return { row, bump: await this.#bump(row.ownerUserId), reaped };
return { row, bump: await this.#bump(row.ownerUserId) };
}
/**
@@ -499,9 +439,8 @@ export class DurableSubscriptionStore extends PuterStore {
});
}
const cached = suspended.filter((row) => !isScheduleRow(row));
await this.stores.eventSubscription.dropDurables(cached);
const owners = new Set(cached.map((row) => row.ownerUserId));
await this.stores.eventSubscription.dropDurables(suspended);
const owners = new Set(suspended.map((row) => row.ownerUserId));
const bumps = await Promise.all(
[...owners].map((owner) => this.#bump(owner)),
);
@@ -529,7 +468,6 @@ export class DurableSubscriptionStore extends PuterStore {
// owner from the first row it is given.
const byOwner = new Map<number, DurableSubscription[]>();
for (const row of rows) {
if (isScheduleRow(row)) continue;
const live: DurableSubscription = {
...row,
suspendedAt: null,
@@ -603,7 +541,7 @@ export class DurableSubscriptionStore extends PuterStore {
row: DurableSubscription,
context: string | null,
): Promise<{ row: DurableSubscription; bump: GenerationBump } | null> {
assertDurableContext(context);
this.#assertContext(context);
const written = await this.clients.db.write(
`UPDATE \`${TABLE}\` SET \`context\` = ? WHERE \`sub_id\` = ?`,
[context, row.subId],
@@ -635,12 +573,8 @@ export class DurableSubscriptionStore extends PuterStore {
* Reap rows past their expiry, dropping each from the cache so the region
* stops delivering against it without waiting for a rebuild.
*/
async sweepExpired(batchSize: number): Promise<DurableReap> {
const cutoff = nowSeconds();
return this.#reap(await this.#listExpired(cutoff, batchSize), {
where: '`expires_at` IS NOT NULL AND `expires_at` <= ?',
values: [cutoff],
});
async sweepExpired(batchSize: number): Promise<number> {
return this.#reap(await this.#listExpired(nowSeconds(), batchSize));
}
/**
@@ -648,14 +582,8 @@ export class DurableSubscriptionStore extends PuterStore {
* never resumes is a row kept only so its holder can see why it stopped,
* and that answer has a shelf life.
*/
async sweepSuspended(
cutoff: number,
batchSize: number,
): Promise<DurableReap> {
return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize), {
where: '`suspended_at` IS NOT NULL AND `suspended_at` <= ?',
values: [cutoff],
});
async sweepSuspended(cutoff: number, batchSize: number): Promise<number> {
return this.#reap(await this.#listSuspendedBefore(cutoff, batchSize));
}
// -- Reads -------------------------------------------------------
@@ -694,16 +622,8 @@ export class DurableSubscriptionStore extends PuterStore {
// The scope half of the predicate is what the total counts over; the
// cursor half only positions one page inside it.
const scope = [
'`holder_user_id` = ?',
this.#unexpiredClause(),
NOT_SCHEDULE_CLAUSE,
];
const scopeParams: unknown[] = [
holderUserId,
nowSeconds(),
SCHEDULE_TOKEN_PATTERN,
];
const scope = ['`holder_user_id` = ?', this.#unexpiredClause()];
const scopeParams: unknown[] = [holderUserId, nowSeconds()];
if (options.appUid !== undefined && options.appUid !== null) {
scope.push('`app_uid` = ?');
scopeParams.push(options.appUid);
@@ -755,8 +675,7 @@ export class DurableSubscriptionStore extends PuterStore {
`SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 0 ` +
'WHEN `app_uid` = ? THEN 1 ELSE 0 END) AS `for_app`, ' +
`SUM(CASE WHEN ${TERMINAL_CLAUSE} THEN 1 ELSE 0 END) AS \`terminal\` ` +
`FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()} ` +
`AND ${NOT_SCHEDULE_CLAUSE}`,
`FROM \`${TABLE}\` WHERE \`holder_user_id\` = ? AND ${this.#unexpiredClause()}`,
[
...terminal,
...terminal,
@@ -764,7 +683,6 @@ export class DurableSubscriptionStore extends PuterStore {
...terminal,
holderUserId,
nowSeconds(),
SCHEDULE_TOKEN_PATTERN,
],
);
return {
@@ -775,26 +693,22 @@ export class DurableSubscriptionStore extends PuterStore {
}
/**
* The holder's live rows, which is what a revocation has to consider, one
* keyset page at a time. Same index the listing and the quota use — passing
* `appUid` narrows to one app's rows, which is what a grant made to that
* app can have authorized. Schedule rows are included: a holder can hold
* far more of them than the durable quota allows, which is why this pages.
* The holder's live rows, which is what a revocation has to consider. Same
* index the listing and the quota use — passing `appUid` narrows to one
* app's rows, which is what a grant made to that app can have authorized.
*
* `includeSuspended` also returns rows out of service for a reason that
* lifts. A revocation needs them: those reasons hold a backlog, and a row
* skipped here comes back in service on the next resume and hands it over.
* Rows suspended for good are left out.
*
* `excludeSchedules` is for a settle no schedule row can match.
* Bounded by the per-account quota, so the whole set fits one read. Rows
* suspended for good hold no slot, so they are left out.
*/
async listActiveForHolderPage(
async listActiveForHolder(
holderUserId: number,
appUid: string | null,
afterId: number,
batchSize: number = HOLDER_SETTLE_BATCH,
opts: { includeSuspended?: boolean; excludeSchedules?: boolean } = {},
): Promise<DurablePage> {
opts: { includeSuspended?: boolean } = {},
): Promise<DurableSubscription[]> {
const where = [
'`holder_user_id` = ?',
opts.includeSuspended
@@ -807,25 +721,21 @@ export class DurableSubscriptionStore extends PuterStore {
...(opts.includeSuspended ? TERMINAL_SUSPENDED_REASONS : []),
nowSeconds(),
];
if (opts.excludeSchedules) {
where.push(NOT_SCHEDULE_CLAUSE);
params.push(SCHEDULE_TOKEN_PATTERN);
}
if (appUid !== null) {
where.push('`app_uid` = ?');
params.push(appUid);
}
return this.#page(where, params, afterId, batchSize, {
primary: true,
});
const rows = await this.clients.db.pread(
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
`WHERE ${where.join(' AND ')} ORDER BY \`id\` LIMIT ?`,
[...params, EVENTS_DURABLE_SUBSCRIPTIONS_MAX],
);
return rows.map(toRow);
}
/**
* The rows bound to one of a scope's handler names. What a removal has to
* The rows bound to one of an app's handler names. What a removal has to
* suspend, and — asking for the suspended half — what a republish resumes.
* `holder` is read for user scope only. App scope never returns user-scope
* rows; user scope is read through the holder index, so it never walks the
* other holders of a popular app.
*
* Bounded by the per-account quota times nothing: a widely-used handler can
* carry more rows than one read should return, so this is a page and the
@@ -833,28 +743,11 @@ export class DurableSubscriptionStore extends PuterStore {
*/
async listByHandler(
appUid: string,
scope: HandlerScopeKind,
holder: number,
handlerName: string,
options: { suspendedReason?: SuspendedReason; limit?: number } = {},
): Promise<DurableSubscription[]> {
const where =
scope === 'user'
? [
'`holder_user_id` = ?',
'`app_uid` = ?',
'`handler_scope` = 1',
'`handler_name` = ?',
]
: [
'`app_uid` = ?',
'`handler_scope` = 0',
'`handler_name` = ?',
];
const params: unknown[] =
scope === 'user'
? [holder, appUid, handlerName]
: [appUid, handlerName];
const where = ['`app_uid` = ?', '`handler_name` = ?'];
const params: unknown[] = [appUid, handlerName];
if (options.suspendedReason === undefined) {
where.push('`suspended_at` IS NULL');
} else {
@@ -875,27 +768,25 @@ export class DurableSubscriptionStore extends PuterStore {
/**
* One holder's rows suspended for a given reason — what a resume condition
* that belongs to the account rather than to a handler releases — one
* keyset page at a time.
* that belongs to the account rather than to a handler releases.
*/
async listSuspendedForHolderPage(
async listSuspendedForHolder(
holderUserId: number,
reason: SuspendedReason,
afterId: number,
batchSize: number = HOLDER_SETTLE_BATCH,
): Promise<DurablePage> {
return this.#page(
): Promise<DurableSubscription[]> {
const rows = await this.clients.db.pread(
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
'WHERE `holder_user_id` = ? AND `suspended_at` IS NOT NULL ' +
`AND \`suspended_reason\` = ? AND ${this.#unexpiredClause()} ` +
'ORDER BY `id` LIMIT ?',
[
'`holder_user_id` = ?',
'`suspended_at` IS NOT NULL',
'`suspended_reason` = ?',
this.#unexpiredClause(),
holderUserId,
reason,
nowSeconds(),
EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
],
[holderUserId, reason, nowSeconds()],
afterId,
batchSize,
{ primary: true },
);
return rows.map(toRow);
}
/**
@@ -928,8 +819,8 @@ export class DurableSubscriptionStore extends PuterStore {
const rows = await this.clients.db.pread(
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
'WHERE `owner_user_id` = ? AND `suspended_at` IS NULL AND ' +
`${this.#unexpiredClause()} AND ${NOT_SCHEDULE_CLAUSE}`,
[ownerUserId, nowSeconds(), SCHEDULE_TOKEN_PATTERN],
`${this.#unexpiredClause()}`,
[ownerUserId, nowSeconds()],
);
return rows.map(toRow);
}
@@ -955,98 +846,30 @@ export class DurableSubscriptionStore extends PuterStore {
* What app deletion reaps: `#reap` already purges the backlog, drops the
* cache and bumps a generation per owner, so this is the whole of it.
*/
async reapForApp(appUid: string, batchSize: number): Promise<DurableReap> {
async reapForApp(appUid: string, batchSize: number): Promise<number> {
return this.#reap(await this.#listForApp(appUid, batchSize));
}
/**
* Delete the row a schedule owns, whether or not the schedule ever recorded
* its id: a create that crashed between the two inserts leaves one it
* didn't. Never cached, so there is nothing to drop or bump. Called by
* extensions.
*/
async removeScheduleSubscription(scheduleUid: string): Promise<boolean> {
const written = await this.clients.db.write(
`DELETE FROM \`${TABLE}\` WHERE \`token\` = ?`,
[scheduleToken(scheduleUid)],
);
return written.anyRowsAffected;
}
// Statements and fragments for the schedules' own transactions and reads,
// so no SQL outside this store names the table. Called by extensions.
/**
* Delete the row a schedule owns, only while `guard` holds: for a batch
* that ends the schedule in the same transaction.
*/
scheduleSubscriptionDeleteEntry(
scheduleUid: string,
guard: SqlGuard,
): BatchEntry {
return {
statement:
`DELETE FROM \`${TABLE}\` WHERE \`token\` = ? ` +
`AND EXISTS (SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where})`,
values: [scheduleToken(scheduleUid), ...guard.values],
};
}
/**
* A `LEFT JOIN` of the row a schedule owns, as `alias`, on `subIdColumn`,
* and the columns it selects: `sub_row_id`, `sub_suspended_at`,
* `sub_suspended_reason`, `sub_handler_name`, `sub_handler_scope` and
* `sub_context`.
*/
scheduleSubscriptionJoin(
alias: string,
subIdColumn: string,
): { columns: string; join: string } {
return {
columns:
`${alias}.\`id\` AS \`sub_row_id\`, ` +
`${alias}.\`suspended_at\` AS \`sub_suspended_at\`, ` +
`${alias}.\`suspended_reason\` AS \`sub_suspended_reason\`, ` +
`${alias}.\`handler_name\` AS \`sub_handler_name\`, ` +
`${alias}.\`handler_scope\` AS \`sub_handler_scope\`, ` +
`${alias}.\`context\` AS \`sub_context\``,
join: `LEFT JOIN \`${TABLE}\` ${alias} ON ${alias}.\`sub_id\` = ${subIdColumn}`,
};
}
/** A condition true when no subscription row has the id `subIdExpr`. */
scheduleSubscriptionAbsent(subIdExpr: string): string {
return `NOT EXISTS (SELECT 1 FROM \`${TABLE}\` e WHERE e.\`sub_id\` = ${subIdExpr})`;
}
// -- Internals ---------------------------------------------------
/**
* Delete a holder's oldest rows suspended for good, past what `list` keeps,
* and return the reap's bumps. Best-effort: the next subscribe tries
* again.
* Delete a holder's oldest rows suspended for good, past what `list` keeps.
* Best-effort: the next subscribe tries again.
*/
async #dropOldestTerminal(
holderUserId: number,
count: number,
): Promise<GenerationBump[]> {
): Promise<void> {
try {
const rows = await this.clients.db.pread(
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
`WHERE \`holder_user_id\` = ? AND ${TERMINAL_CLAUSE} ` +
`AND ${NOT_SCHEDULE_CLAUSE} ` +
'ORDER BY `suspended_at`, `id` LIMIT ?',
[
holderUserId,
...TERMINAL_SUSPENDED_REASONS,
SCHEDULE_TOKEN_PATTERN,
count,
],
[holderUserId, ...TERMINAL_SUSPENDED_REASONS, count],
);
return (await this.#reap(rows.map(toRow))).bumps;
await this.#reap(rows.map(toRow));
} catch (err) {
console.warn('[events] could not drop old revoked rows', err);
return [];
}
}
@@ -1068,23 +891,15 @@ export class DurableSubscriptionStore extends PuterStore {
return rows.map(toRow);
}
/**
* One keyset page, ordered and positioned by primary key. The replica
* unless `primary`, which settles need: a lagging read would skip a row
* just created.
*/
/** One page of a whole-table scan, ordered and positioned by primary key. */
async #page(
where: string[],
params: unknown[],
afterId: number,
batchSize: number,
opts: { primary?: boolean } = {},
): Promise<DurablePage> {
const limit = Math.max(1, Math.floor(batchSize));
const read = opts.primary
? this.clients.db.pread.bind(this.clients.db)
: this.clients.db.read.bind(this.clients.db);
const rows = await read(
const rows = await this.clients.db.read(
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
`WHERE ${[...where, '`id` > ?'].join(' AND ')} ` +
'ORDER BY `id` LIMIT ?',
@@ -1137,56 +952,30 @@ export class DurableSubscriptionStore extends PuterStore {
return rows.map(toRow);
}
/**
* Delete a batch and stop this region delivering against any of it. Other
* regions stop once the caller publishes the returned bumps; the deleted
* rows come back so the caller can end the schedules among them.
*
* `stillDue` re-checks candidates read from a replica on the primary, so a
* row resumed or extended since is kept and not reported.
*/
async #reap(
candidates: readonly DurableSubscription[],
stillDue?: { where: string; values: unknown[] },
): Promise<DurableReap> {
if (candidates.length === 0) return { rows: [], bumps: [] };
/** Delete a batch and leave no region delivering against any of it. */
async #reap(rows: readonly DurableSubscription[]): Promise<number> {
if (rows.length === 0) return 0;
const ids = candidates.map((row) => row.subId);
const inList = `(${ids.map(() => '?').join(', ')})`;
await this.clients.db.write(
`DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}` +
(stillDue ? ` AND ${stillDue.where}` : ''),
[...ids, ...(stillDue?.values ?? [])],
`DELETE FROM \`${TABLE}\` WHERE \`sub_id\` IN ` +
`(${rows.map(() => '?').join(', ')})`,
rows.map((row) => row.subId),
);
let rows = candidates;
if (stillDue) {
const kept = new Set(
(
await this.clients.db.pread(
`SELECT \`sub_id\` FROM \`${TABLE}\` WHERE \`sub_id\` IN ${inList}`,
ids,
)
).map((row) => String(row.sub_id)),
);
rows = candidates.filter((row) => !kept.has(row.subId));
if (rows.length === 0) return { rows: [], bumps: [] };
}
const cached = rows.filter((row) => !isScheduleRow(row));
await this.stores.eventSubscription.dropDurables(cached);
for (const row of cached) {
await this.stores.eventSubscription.dropDurables(rows);
for (const row of rows) {
// Whatever was still owed to a row that no longer exists.
try {
await this.stores.pendingDelivery.purge(row.subId);
} catch {
// The pending sweep purges it once it finds the row gone.
}
await this.stores.pendingDelivery.purge(row.subId).catch(() => {});
}
const owners = new Set(cached.map((row) => row.ownerUserId));
const bumps: GenerationBump[] = [];
for (const ownerUserId of owners)
bumps.push(await this.#bump(ownerUserId));
return { rows: [...rows], bumps };
const owners = new Set(rows.map((row) => row.ownerUserId));
for (const ownerUserId of owners) await this.#bump(ownerUserId);
return rows.length;
}
#assertContext(context: string | null): void {
if (context === null) return;
if (Buffer.byteLength(context, 'utf8') > DURABLE_CONTEXT_MAX_BYTES)
throw contextTooLarge();
}
async #bump(ownerUserId: number): Promise<GenerationBump> {
File diff suppressed because it is too large. Load diff
@@ -1,392 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The user-scope handler schema and queries against MySQL 8.4. Runs only when
* `PUTER_TEST_MYSQL_PORT` points at a server (root / `puter`).
*/
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import mysql from 'mysql2/promise';
import { v4 as uuidv4 } from 'uuid';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { splitMysqlStatements } from '../../clients/database/splitMysqlStatements.js';
import type { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import { appHandlerScope, hashContent } from './EventHandlerStore.js';
const connection = {
host: '127.0.0.1',
port: Number(process.env.PUTER_TEST_MYSQL_PORT ?? 3307),
user: 'root',
password: 'puter',
};
const database = `event_user_handlers_${Date.now().toString(36)}`;
const migrationDir = fileURLToPath(
new URL('../../clients/database/migrations/mysql', import.meta.url),
);
const SOURCE = 'async ({ event }) => { console.log(event.path); }';
describe.skipIf(!process.env.PUTER_TEST_MYSQL_PORT)(
'user-scoped handlers (MySQL)',
() => {
let server: PuterServer;
let conn: mysql.Connection;
const userIds: number[] = [];
const handlers = () => server.stores.eventHandler;
const rows = async (sql: string, params: unknown[] = []) =>
(await conn.query(sql, params))[0] as Record<string, unknown>[];
const applyMig55 = async () => {
const sql = readFileSync(
`${migrationDir}/mysql_mig_55.sql`,
'utf8',
);
for (const statement of splitMysqlStatements(sql))
await conn.query(statement);
};
const bind = async (
holder: number,
app: string,
name: string,
handlerScope: number,
suspended = false,
) => {
await conn.query(
'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, `holder_user_id`, `app_uid`, `subject`, `anchor_uid`, `anchor_path`, `delivery`, `handler_name`, `handler_scope`, `targets`, `permission`, `suspended_at`, `suspended_reason`, `created_at`) ' +
"VALUES (?, ?, ?, ?, ?, 'fs:/x', 'a', '/x', 'single', ?, ?, '[]', 'list', ?, ?, 1)",
[
uuidv4(),
`f#${uuidv4()}`,
holder,
holder,
app,
name,
handlerScope,
suspended ? 1 : null,
suspended ? 'handler_not_found' : null,
],
);
};
/**
* The `key` column of every table in the plan of the last statement a
* spy saw.
*/
const planKeys = async (spy: {
mock: { calls: unknown[][] };
}): Promise<Record<string, string | null>> => {
const [statement, params] = spy.mock.calls.at(-1) as [
string,
unknown[],
];
const plan = await rows(`EXPLAIN ${statement}`, params);
return Object.fromEntries(
plan.map((row) => [
String(row.table),
(row.key as string | null) ?? null,
]),
);
};
beforeAll(async () => {
const admin = await mysql.createConnection(connection);
await admin.query(`CREATE DATABASE \`${database}\``);
await admin.end();
server = await setupTestServer({
database: {
engine: 'mysql',
...connection,
database,
migrationPaths: [migrationDir],
},
events: { enabled: true },
} as never);
conn = await mysql.createConnection({ ...connection, database });
for (let i = 0; i < 12; i++) {
const user = await server.stores.user.create({
username: `ush-${uuidv4().slice(0, 8)}`,
uuid: uuidv4(),
password: null,
email: null,
});
userIds.push(user.id);
}
}, 240_000);
afterAll(async () => {
await conn?.end();
await server?.shutdown();
const admin = await mysql.createConnection(connection);
await admin.query(`DROP DATABASE IF EXISTS \`${database}\``);
await admin.end();
});
it('runs on MySQL 8.4', async () => {
const [row] = await rows('SELECT VERSION() AS version');
expect(String(row!.version)).toMatch(/^8\.4\./);
});
it('applies over existing subscription rows, and a replay is a no-op', async () => {
// Back to the state before the migration, with rows already in.
await conn.query('DROP TABLE `event_user_handlers`');
await conn.query(
'ALTER TABLE `event_subscriptions` DROP COLUMN `handler_scope`',
);
await conn.query('DELETE FROM `event_subscriptions`');
for (const name of ['a', 'b', 'c'])
await conn.query(
'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, `holder_user_id`, `app_uid`, `subject`, `anchor_uid`, `anchor_path`, `delivery`, `handler_name`, `targets`, `permission`, `created_at`) ' +
"VALUES (?, ?, ?, ?, 'app-old', 'fs:/x', 'a', '/x', 'single', ?, '[]', 'list', 1)",
[uuidv4(), `f#${uuidv4()}`, userIds[0], userIds[0], name],
);
await applyMig55();
const scopes = async () =>
(
await rows(
'SELECT `handler_scope` FROM `event_subscriptions`',
)
).map((row) => Number(row.handler_scope));
expect(await scopes()).toEqual([0, 0, 0]);
await applyMig55();
expect(await scopes()).toEqual([0, 0, 0]);
const [column] = await rows(
"SELECT COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'event_subscriptions' AND COLUMN_NAME = 'handler_scope'",
);
expect(column).toMatchObject({
COLUMN_TYPE: 'tinyint unsigned',
IS_NULLABLE: 'NO',
COLUMN_DEFAULT: '0',
});
const indexes = await rows(
"SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'event_user_handlers' ORDER BY INDEX_NAME, SEQ_IN_INDEX",
);
const byIndex = (name: string) =>
indexes
.filter((row) => row.INDEX_NAME === name)
.map((row) => String(row.COLUMN_NAME));
expect(byIndex('idx_event_user_handlers_app_holder_name')).toEqual([
'app_uid',
'holder_user_id',
'name',
]);
expect(byIndex('idx_event_user_handlers_holder_app')).toEqual([
'holder_user_id',
'app_uid',
]);
expect(
indexes.find(
(row) =>
row.INDEX_NAME ===
'idx_event_user_handlers_app_holder_name',
),
).toMatchObject({ NON_UNIQUE: 0 });
const [appUidColumn] = await rows(
"SELECT CHARACTER_SET_NAME, COLLATION_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'event_user_handlers' AND COLUMN_NAME = 'app_uid'",
);
expect(appUidColumn).toMatchObject({
CHARACTER_SET_NAME: 'latin1',
COLLATION_NAME: 'latin1_swedish_ci',
});
});
it('keeps the same name once in the app scope and once per holder', async () => {
const app = `app-${uuidv4()}`;
const insert = (holder: number) =>
conn.query(
'INSERT INTO `event_user_handlers` (`app_uid`, `holder_user_id`, `name`, `source`, `source_hash`, `created_at`, `updated_at`, `scope_since`) VALUES (?, ?, ?, ?, ?, 1, 1, 1)',
[app, holder, 'daily', SOURCE, hashContent(SOURCE)],
);
await handlers().publish({
scope: appHandlerScope(app),
name: 'daily',
source: SOURCE,
});
await insert(userIds[1]!);
await insert(userIds[2]!);
await expect(insert(userIds[1]!)).rejects.toMatchObject({
code: 'ER_DUP_ENTRY',
});
});
it('runs the store against both tables', async () => {
const app = `app-${uuidv4()}`;
const [a, b] = [userIds[3]!, userIds[4]!];
const scope = (holder: number) => ({ appUid: app, holder });
await handlers().publish({
scope: appHandlerScope(app),
name: 'daily',
source: SOURCE,
});
for (const holder of [a, b])
await handlers().publish({
scope: scope(holder),
name: 'daily',
source: `${SOURCE} // ${holder}`,
});
await handlers().publish({
scope: scope(a),
name: `@${hashContent(SOURCE)}`,
source: SOURCE,
});
expect(await handlers().countForApp(scope(a))).toBe(2);
expect(await handlers().countForApp(appHandlerScope(app))).toBe(1);
expect(await handlers().totalSourceBytesForApp(scope(b))).toBe(
Buffer.byteLength(`${SOURCE} // ${b}`),
);
expect(
(await handlers().setForApp(scope(a), { replica: true })).map(
(h) => h.name,
),
).toEqual([`@${hashContent(SOURCE)}`, 'daily']);
expect(await handlers().countUserScopesWithHandlersForUser(a)).toBe(
1,
);
expect(
await handlers().countUserScopesWithHandlersForUser(a, {
createdBefore: 1_000,
}),
).toBe(0);
expect(
await handlers().countUserScopesWithHandlersForUser(a, {
createdBefore: Date.now() + 60_000,
}),
).toBe(1);
const page = await handlers().listUserScopesForUser(a);
expect(page.total).toBe(1);
expect(page.items).toEqual([
expect.objectContaining({ appUid: app, handlerCount: 2 }),
]);
expect(
(await handlers().listUserScopes(app)).items.map(
(s) => s.holder,
),
).toEqual([a, b].sort((x, y) => x - y));
await bind(a, app, 'daily', 1);
await bind(b, app, 'daily', 1, true);
await bind(a, app, 'daily', 0);
expect(
await handlers().countSubscriptionsByHandler(scope(a)),
).toEqual(new Map([['daily', 1]]));
expect(
await handlers().countSubscriptionsByHandler(
appHandlerScope(app),
),
).toEqual(new Map([['daily', 1]]));
await conn.query(
'UPDATE `event_user_handlers` SET `updated_at` = 1 WHERE `app_uid` = ?',
[app],
);
const found = await handlers().listUnboundUserHandlers(1_000);
expect(
found.items
.filter((item) => item.appUid === app)
.map((item) => [item.holder, item.name]),
).toEqual([[a, `@${hashContent(SOURCE)}`]]);
await expect(
handlers().deleteUserHandlerIfUnbound(scope(a), 'daily', 1_000),
).resolves.toBe(false);
await expect(
handlers().deleteUserHandlerIfUnbound(
scope(a),
`@${hashContent(SOURCE)}`,
1_000,
),
).resolves.toBe(true);
await expect(handlers().deleteUserScopesForHolder(b)).resolves.toBe(
1,
);
await expect(handlers().deleteForApp(app)).resolves.toBe(2);
expect(await handlers().countForApp(scope(a))).toBe(0);
});
it('serves the user-scope reads from the holder-leading indexes', async () => {
const app = `app-${uuidv4()}`;
// A popular handler: many holders on one (app, name), so the
// (app_uid, handler_name) index is the wrong one to pick.
for (const holder of userIds)
for (let i = 0; i < 80; i++)
await bind(holder, app, 'popular', 1);
await handlers().publish({
scope: { appUid: app, holder: userIds[0]! },
name: 'popular',
source: SOURCE,
});
await conn.query('ANALYZE TABLE `event_subscriptions`');
await conn.query('ANALYZE TABLE `event_user_handlers`');
const pread = vi.spyOn(server.clients.db, 'pread');
const read = vi.spyOn(server.clients.db, 'read');
try {
await server.stores.durableSubscription.listByHandler(
app,
'user',
userIds[0]!,
'popular',
);
expect(await planKeys(pread)).toEqual({
event_subscriptions: 'idx_event_subscriptions_holder',
});
await handlers().countSubscriptionsByHandler({
appUid: app,
holder: userIds[0]!,
});
expect(await planKeys(read)).toEqual({
event_subscriptions: 'idx_event_subscriptions_holder',
});
await handlers().countUserScopesWithHandlersForUser(
userIds[0]!,
{
createdBefore: Date.now(),
},
);
expect((await planKeys(pread)).event_user_handlers).toBe(
'idx_event_user_handlers_holder_app',
);
await handlers().listUserScopesForUser(userIds[0]!);
expect((await planKeys(read)).event_user_handlers).toBe(
'idx_event_user_handlers_holder_app',
);
await handlers().listUnboundUserHandlers(1_000);
expect((await planKeys(read)).s).toBe(
'idx_event_subscriptions_holder',
);
} finally {
pread.mockRestore();
read.mockRestore();
}
}, 120_000);
},
);
+87 -593
View File
@@ -21,7 +21,6 @@ import { createHash } from 'node:crypto';
import {
EVENTS_HANDLERS_PER_APP,
EVENTS_HANDLER_SOURCE_MAX_BYTES,
EVENTS_USER_HANDLERS_PER_SCOPE,
} from '../../controllers/events/limits.js';
import { HttpError } from '../../core/http/HttpError.js';
import {
@@ -48,7 +47,6 @@ import { PuterStore } from '../types.js';
*/
const TABLE = 'event_handlers';
const USER_TABLE = 'event_user_handlers';
const SUBSCRIPTION_TABLE = 'event_subscriptions';
const APP_TABLE = 'apps';
@@ -59,24 +57,12 @@ const APP_TABLE = 'apps';
export const EVENTS_WORKERS_LIST_DEFAULT_LIMIT = 50;
export const EVENTS_WORKERS_LIST_LIMIT_CAP = 200;
/** Default page of `listUserScopes`, and of one GC read. */
const USER_SCOPES_PAGE_DEFAULT = 500;
const USER_SCOPES_PAGE_CAP = 1000;
const UNBOUND_HANDLERS_BATCH = 200;
/** Longest a handler name may be, matching the column that holds it. */
export const HANDLER_NAME_MAX_LENGTH = 128;
/** Names are addressable identifiers, not free text. */
const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
/**
* A server-minted name for an inline handler: `@` and the source hash. `@` is
* outside {@link NAME_PATTERN}, so a minted name never collides with a chosen
* one, and only a user scope accepts it.
*/
const INLINE_NAME_PATTERN = /^@[0-9a-f]{64}$/;
/**
* Longest a delivery may address an app's handler set from cache. Every write
* here drops the entry on every node, so this only bounds a read that raced a
@@ -84,66 +70,8 @@ const INLINE_NAME_PATTERN = /^@[0-9a-f]{64}$/;
*/
export const HANDLER_SET_CACHE_TTL_SECONDS = 30;
// -- Scopes -----------------------------------------------------------
/**
* Whose handlers a call is about: an app's own set (`holder` 0), or the
* handlers one user published into their scope of that app. The two live in
* separate tables, so code that predates user scopes never reads one.
*/
export interface HandlerScope {
appUid: string;
/** `users.id` of the scope's holder, or 0 for the app's own set. */
holder: number;
}
/** The app's own scope. */
export const appHandlerScope = (appUid: string): HandlerScope => ({
appUid,
holder: 0,
});
const isUserScope = (scope: HandlerScope): boolean => scope.holder !== 0;
const assertScope = (scope: HandlerScope): void => {
if (!Number.isInteger(scope.holder) || scope.holder < 0)
throw new TypeError('handler scope holder must be a user id or 0');
};
const tableOf = (scope: HandlerScope): string =>
isUserScope(scope) ? USER_TABLE : TABLE;
const handlerCapOf = (scope: HandlerScope): number =>
isUserScope(scope)
? EVENTS_USER_HANDLERS_PER_SCOPE
: EVENTS_HANDLERS_PER_APP;
/** The WHERE clause for a scope's rows and its parameters. */
const scopeFilter = (
scope: HandlerScope,
): { where: string; params: unknown[] } => {
assertScope(scope);
return isUserScope(scope)
? {
where: '`app_uid` = ? AND `holder_user_id` = ?',
params: [scope.appUid, scope.holder],
}
: { where: '`app_uid` = ?', params: [scope.appUid] };
};
// App scope keeps the key it always had, so nodes on either side of a rolling
// deploy share it.
const handlerSetCacheKey = (scope: HandlerScope): string =>
isUserScope(scope)
? `eventHandlers:uset:${scope.appUid}:${scope.holder}`
: `eventHandlers:set:${scope.appUid}`;
/**
* The same set as read from the replica, kept apart so a lagging read can never
* stand in for the primary's answer on the delivery path.
*/
const replicaSetCacheKey = (scope: HandlerScope): string =>
`${handlerSetCacheKey(scope)}:r`;
const handlerSetCacheKey = (appUid: string): string =>
`eventHandlers:set:${appUid}`;
// -- Shapes -----------------------------------------------------------
@@ -191,45 +119,8 @@ export interface ListEventsWorkersOptions {
appUid?: string;
}
/** One user scope: the handlers a user keeps in one app. */
export interface UserScopeSummary {
appUid: string;
/** Empty when the app row is gone. */
appName: string;
appTitle: string;
handlerCount: number;
/** Earliest handler `created_at` in the scope. */
createdAt: number;
/** Latest handler `updated_at` in the scope. */
updatedAt: number;
}
/** A user scope as an app-wide listing reports it. */
export interface AppUserScope {
holder: number;
handlerCount: number;
createdAt: number;
updatedAt: number;
}
/** One user-scope handler a GC pass may delete. */
export interface UnboundUserHandler {
id: number;
appUid: string;
holder: number;
name: string;
}
/**
* One entry of a scope's handler set. `updatedAt` (unix seconds) is set for a
* user scope and absent from entries cached without it, meaning unknown.
*/
export type HandlerSetEntry = Pick<EventHandler, 'name' | 'sourceHash'> & {
updatedAt?: number;
};
export interface PublishHandlerInput {
scope: HandlerScope;
appUid: string;
name: string;
source: string;
/**
@@ -239,11 +130,6 @@ export interface PublishHandlerInput {
ifHash?: string | null;
/** Take the name whatever is published under it. */
replace?: boolean;
/**
* A create leaves peer nodes' cached sets alone: one missing the new name
* is re-read when asked for it. Updates always broadcast.
*/
broadcast?: boolean;
}
/** What one publish did, which is what the caller reports back. */
@@ -276,12 +162,10 @@ const invalidSource = (): HttpError =>
legacyCode: 'events_handler_source_invalid',
});
const tooManyHandlers = (scope: HandlerScope): HttpError =>
const tooManyHandlers = (): HttpError =>
new HttpError(
429,
isUserScope(scope)
? `A user scope may hold ${EVENTS_USER_HANDLERS_PER_SCOPE} handlers`
: `An app may publish ${EVENTS_HANDLERS_PER_APP} handlers`,
`An app may publish ${EVENTS_HANDLERS_PER_APP} handlers`,
{ legacyCode: 'events_handler_limit' },
);
@@ -320,11 +204,10 @@ const SELECT_COLUMNS =
export const hashContent = (content: string): string =>
createHash('sha256').update(content, 'utf8').digest('hex');
/** `userScope` also admits the `@<sha256>` names minted for inline handlers. */
export const isValidHandlerName = (name: string, userScope = false): boolean =>
export const isValidHandlerName = (name: string): boolean =>
name.length > 0 &&
name.length <= HANDLER_NAME_MAX_LENGTH &&
(NAME_PATTERN.test(name) || (userScope && INLINE_NAME_PATTERN.test(name)));
NAME_PATTERN.test(name);
export class EventHandlerStore extends PuterStore {
// -- Writes ------------------------------------------------------
@@ -337,12 +220,11 @@ export class EventHandlerStore extends PuterStore {
* with a row that no longer exists.
*/
async publish(input: PublishHandlerInput): Promise<PublishResult> {
const { scope } = input;
const name = this.#assertName(input.name, scope);
const name = this.#assertName(input.name);
const source = this.#assertSource(input.source);
const sourceHash = hashContent(source);
const existing = await this.getByName(scope, name);
const existing = await this.getByName(input.appUid, name);
if (!existing) {
// `ifHash` naming a source that is not there is the same race from
// the other side: something removed the row this publish was
@@ -350,11 +232,10 @@ export class EventHandlerStore extends PuterStore {
if (input.ifHash && !input.replace) throw publishConflict(name);
return {
handler: await this.#insert(
scope,
input.appUid,
name,
source,
sourceHash,
input.broadcast !== false,
),
outcome: 'created',
};
@@ -372,155 +253,34 @@ export class EventHandlerStore extends PuterStore {
if (!input.replace && !basedOnPublished) throw publishConflict(name);
return {
handler: await this.#update(scope, existing, source, sourceHash),
handler: await this.#update(existing, source, sourceHash),
outcome: 'updated',
};
}
/** Drop one handler. Null when the scope had nothing published by that name. */
async remove(
scope: HandlerScope,
name: string,
): Promise<EventHandler | null> {
const existing = await this.getByName(scope, name);
/** Drop one handler. Null when the app had nothing published by that name. */
async remove(appUid: string, name: string): Promise<EventHandler | null> {
const existing = await this.getByName(appUid, name);
if (!existing) return null;
const { where, params } = scopeFilter(scope);
await this.clients.db.write(
`DELETE FROM \`${tableOf(scope)}\` WHERE ${where} AND \`name\` = ?`,
[...params, name],
`DELETE FROM \`${TABLE}\` WHERE \`app_uid\` = ? AND \`name\` = ?`,
[appUid, name],
);
await this.#forgetSet(scope);
await this.#forgetSet(appUid);
return existing;
}
/**
* Drop every handler an app has published, in its own set and in every
* user's scope — what a deleted app leaves behind. Returns the rows
* removed. User scopes go a batch of holders at a time, each batch's cached
* sets dropped here and on every peer.
* Drop every handler an app has published — what a deleted app leaves
* behind.
*/
async deleteForApp(appUid: string): Promise<number> {
const result = await this.clients.db.write(
`DELETE FROM \`${TABLE}\` WHERE \`app_uid\` = ?`,
[appUid],
);
let removed = result.affectedRows ?? 0;
for (;;) {
const holders = (
await this.clients.db.pread(
`SELECT DISTINCT \`holder_user_id\` FROM \`${USER_TABLE}\` ` +
'WHERE `app_uid` = ? ORDER BY `holder_user_id` LIMIT ?',
[appUid, USER_SCOPES_PAGE_DEFAULT],
)
).map((row) => Number(row.holder_user_id));
if (holders.length === 0) break;
const batch = await this.clients.db.write(
`DELETE FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? ` +
`AND \`holder_user_id\` IN (${holders.map(() => '?').join(', ')})`,
[appUid, ...holders],
);
await this.#forgetSets(
holders.map((holder) => ({ appUid, holder })),
);
const affected = batch.affectedRows ?? 0;
removed += affected;
if (affected === 0) break;
}
await this.#forgetSet(appHandlerScope(appUid));
return removed;
}
/** Drop one user's scope of one app. Returns the handlers removed. */
async deleteUserScope(appUid: string, holder: number): Promise<number> {
const scope = { appUid, holder };
if (!isUserScope(scope)) return 0;
assertScope(scope);
const result = await this.clients.db.write(
`DELETE FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` = ?`,
[appUid, holder],
);
await this.#forgetSet(scope);
return result.affectedRows ?? 0;
}
/**
* Drop every scope a user holds, across apps — what an account deletion
* leaves behind. Returns the handlers removed.
*/
async deleteUserScopesForHolder(userId: number): Promise<number> {
if (!Number.isInteger(userId) || userId <= 0) return 0;
const apps = await this.clients.db.pread(
`SELECT DISTINCT \`app_uid\` FROM \`${USER_TABLE}\` WHERE \`holder_user_id\` = ?`,
[userId],
);
const result = await this.clients.db.write(
`DELETE FROM \`${USER_TABLE}\` WHERE \`holder_user_id\` = ?`,
[userId],
);
for (const row of apps)
await this.#forgetSet({
appUid: String(row.app_uid),
holder: userId,
});
return result.affectedRows ?? 0;
}
/**
* Delete one user-scope handler if no subscription row of any state is
* bound to it and it was last touched before `olderThan` (unix seconds).
* The check and the delete are one statement, so a row bound in between
* keeps the handler. True when it deleted.
*/
async deleteUserHandlerIfUnbound(
scope: HandlerScope,
name: string,
olderThan: number,
): Promise<boolean> {
if (!isUserScope(scope)) return false;
assertScope(scope);
const result = await this.clients.db.write(
`DELETE FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` = ? ` +
'AND `name` = ? AND `updated_at` < ? ' +
`AND NOT EXISTS (SELECT 1 FROM \`${SUBSCRIPTION_TABLE}\` ` +
'WHERE `holder_user_id` = ? AND `app_uid` = ? ' +
'AND `handler_scope` = 1 AND `handler_name` = ?)',
[
scope.appUid,
scope.holder,
name,
olderThan,
scope.holder,
scope.appUid,
name,
],
);
if (!result.anyRowsAffected) return false;
await this.#forgetSet(scope);
return true;
}
/**
* Mark one user-scope handler as just used, so the unbound sweep (which
* only takes rows last touched before its cutoff) leaves it alone. False
* when the handler is gone.
*/
async touchUserHandler(
scope: HandlerScope,
name: string,
): Promise<boolean> {
if (!isUserScope(scope)) return false;
assertScope(scope);
const result = await this.clients.db.write(
`UPDATE \`${USER_TABLE}\` SET \`updated_at\` = ? ` +
'WHERE `app_uid` = ? AND `holder_user_id` = ? AND `name` = ?',
[nowSeconds(), scope.appUid, scope.holder, name],
);
return Boolean(result.anyRowsAffected);
}
/** Drop this region's cached sets of a scope, without telling peers. */
async forgetScopeCache(scope: HandlerScope): Promise<void> {
await this.#forgetSet(scope, false);
await this.#forgetSet(appUid);
return result.affectedRows;
}
// -- Reads -------------------------------------------------------
@@ -532,36 +292,34 @@ export class EventHandlerStore extends PuterStore {
* absent.
*/
async getByName(
scope: HandlerScope,
appUid: string,
name: string,
): Promise<EventHandler | null> {
const { where, params } = scopeFilter(scope);
const rows = await this.clients.db.pread(
`SELECT ${SELECT_COLUMNS} FROM \`${tableOf(scope)}\` ` +
`WHERE ${where} AND \`name\` = ?`,
[...params, name],
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
'WHERE `app_uid` = ? AND `name` = ?',
[appUid, name],
);
return rows.length > 0 ? toRow(rows[0]) : null;
}
/**
* What a scope has published, with how many subscriptions each name is
* carrying. Bounded by the scope's cap, so the whole set is one page.
* What an app has published, with how many subscriptions each name is
* carrying. Bounded by the per-app cap, so the whole set is one page.
*
* The counts come from one grouped read rather than a join, because the
* subscription side has to count names a scope never published — a handler
* subscription side has to count names an app never published — a handler
* removed while subscriptions still pointed at it leaves exactly that.
*/
async listForApp(scope: HandlerScope): Promise<EventHandlerSummary[]> {
const { where, params } = scopeFilter(scope);
async listForApp(appUid: string): Promise<EventHandlerSummary[]> {
const rows = await this.clients.db.read(
'SELECT `name`, `source_hash`, `updated_at` ' +
`FROM \`${tableOf(scope)}\` WHERE ${where} ` +
`FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
'ORDER BY `name` LIMIT ?',
[...params, handlerCapOf(scope)],
[appUid, EVENTS_HANDLERS_PER_APP],
);
const counts = await this.countSubscriptionsByHandler(scope);
const counts = await this.countSubscriptionsByHandler(appUid);
return rows.map((row) => ({
name: String(row.name),
hash: String(row.source_hash),
@@ -571,30 +329,24 @@ export class EventHandlerStore extends PuterStore {
}
/**
* Name and source hash for every handler a scope has published: the
* identity of its set, without reading a byte of source. What a deploy
* checks the script it was asked for against, and what the delivery path's
* cache is filled from. Primary, so neither reads a set a replica has not
* caught up to.
* Name and source hash for every handler an app has published: the identity
* of its set, without reading a byte of source. What a deploy checks the
* script it was asked for against, and what the delivery path's cache is
* filled from. Primary, so neither reads a set a replica has not caught up
* to.
*/
async setForApp(
scope: HandlerScope,
options: { replica?: boolean } = {},
): Promise<HandlerSetEntry[]> {
const { where, params: whereParams } = scopeFilter(scope);
const withAge = isUserScope(scope);
const sql =
`SELECT \`name\`, \`source_hash\`${withAge ? ', `updated_at`' : ''} ` +
`FROM \`${tableOf(scope)}\` WHERE ${where} ` +
'ORDER BY `name` LIMIT ?';
const params = [...whereParams, handlerCapOf(scope)];
const rows = options.replica
? await this.clients.db.read(sql, params)
: await this.clients.db.pread(sql, params);
appUid: string,
): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
const rows = await this.clients.db.pread(
'SELECT `name`, `source_hash` ' +
`FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` +
'ORDER BY `name` LIMIT ?',
[appUid, EVENTS_HANDLERS_PER_APP],
);
return rows.map((row) => ({
name: String(row.name),
sourceHash: String(row.source_hash),
...(withAge ? { updatedAt: Number(row.updated_at) || 0 } : {}),
}));
}
@@ -602,43 +354,23 @@ export class EventHandlerStore extends PuterStore {
* `setForApp` for the delivery path, which asks once per invocation. Cached
* for up to {@link HANDLER_SET_CACHE_TTL_SECONDS}, and dropped everywhere by
* every write here. An empty set is not cached: it fails the delivery, and
* must not outlast the publish that fills it. Schedule runs read a miss
* from the replica (`missRead: 'replica'`), since they come by the clock
* rather than after a write, and cache it under a key of their own; an
* empty replica read is checked on the primary, since a lagging replica
* right after a publish would fail the run. Called by extensions.
* must not outlast the publish that fills it.
*/
async cachedSetForApp(
scope: HandlerScope,
options: {
missRead?: 'primary' | 'replica';
handlerName?: string;
} = {},
): Promise<HandlerSetEntry[]> {
const replica = options.missRead === 'replica';
// A set without the handler may be from before its publish.
const usable = (set: Pick<EventHandler, 'name'>[]) =>
set.length > 0 &&
(options.handlerName === undefined ||
set.some((handler) => handler.name === options.handlerName));
const key = replica
? replicaSetCacheKey(scope)
: handlerSetCacheKey(scope);
appUid: string,
): Promise<Pick<EventHandler, 'name' | 'sourceHash'>[]> {
const key = handlerSetCacheKey(appUid);
try {
const cached = JSON.parse(
(await this.clients.redis.get(key)) ?? 'null',
) as unknown;
if (
Array.isArray(cached) &&
usable(cached as Pick<EventHandler, 'name'>[])
)
return cached as HandlerSetEntry[];
if (Array.isArray(cached) && cached.length > 0)
return cached as Pick<EventHandler, 'name' | 'sourceHash'>[];
} catch {
// Read the table instead.
}
let set = await this.setForApp(scope, { replica });
if (replica && !usable(set)) set = await this.setForApp(scope);
const set = await this.setForApp(appUid);
if (set.length > 0) {
try {
await this.clients.redis.set(
@@ -655,70 +387,57 @@ export class EventHandlerStore extends PuterStore {
}
/**
* Every handler the scope has published, sources included — what the events
* Every handler the app has published, sources included — what the events
* worker is generated from. Primary, for the same reason `getByName` is:
* the bake follows the publish that triggered it.
*/
async allForApp(scope: HandlerScope): Promise<EventHandler[]> {
const { where, params } = scopeFilter(scope);
async allForApp(appUid: string): Promise<EventHandler[]> {
const rows = await this.clients.db.pread(
`SELECT ${SELECT_COLUMNS} FROM \`${tableOf(scope)}\` ` +
`WHERE ${where} ORDER BY \`name\` LIMIT ?`,
[...params, handlerCapOf(scope)],
`SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` +
'WHERE `app_uid` = ? ORDER BY `name` LIMIT ?',
[appUid, EVENTS_HANDLERS_PER_APP],
);
return rows.map(toRow);
}
async countForApp(scope: HandlerScope): Promise<number> {
const { where, params } = scopeFilter(scope);
async countForApp(appUid: string): Promise<number> {
const [row] = await this.clients.db.pread(
`SELECT COUNT(*) AS \`total\` FROM \`${tableOf(scope)}\` WHERE ${where}`,
params,
`SELECT COUNT(*) AS \`total\` FROM \`${TABLE}\` WHERE \`app_uid\` = ?`,
[appUid],
);
return Number(row?.total ?? 0);
}
/**
* Bytes of source a scope has published, across every handler. What a
* publish is checked against before it lands, so the generated events
* Bytes of source an app has published, across every handler. What a
* publish is checked against before it lands, so the app's generated events
* worker never grows past what the runtime may load.
*
* `OCTET_LENGTH`, not `LENGTH`: only MySQL measures the latter in bytes,
* and the caller compares this against `Buffer.byteLength`.
*/
async totalSourceBytesForApp(scope: HandlerScope): Promise<number> {
const { where, params } = scopeFilter(scope);
async totalSourceBytesForApp(appUid: string): Promise<number> {
const [row] = await this.clients.db.pread(
`SELECT SUM(OCTET_LENGTH(\`source\`)) AS \`total\` FROM \`${tableOf(scope)}\` WHERE ${where}`,
params,
`SELECT SUM(OCTET_LENGTH(\`source\`)) AS \`total\` FROM \`${TABLE}\` WHERE \`app_uid\` = ?`,
[appUid],
);
return Number(row?.total ?? 0);
}
/**
* How many subscriptions each of a scope's handler names is carrying,
* How many subscriptions each of an app's handler names is carrying,
* suspended rows included — a suspended subscription is still a dependent,
* and it is the reason a removal is not a delete.
*
* A user scope reads by holder, so it never scans the other holders of a
* popular app.
*/
async countSubscriptionsByHandler(
scope: HandlerScope,
appUid: string,
): Promise<Map<string, number>> {
assertScope(scope);
const filter = isUserScope(scope)
? '`holder_user_id` = ? AND `app_uid` = ? AND `handler_scope` = 1'
: '`app_uid` = ? AND `handler_scope` = 0';
const params = isUserScope(scope)
? [scope.holder, scope.appUid]
: [scope.appUid];
const rows = await this.clients.db.read(
'SELECT `handler_name`, COUNT(*) AS `total` ' +
`FROM \`${SUBSCRIPTION_TABLE}\` ` +
`WHERE ${filter} AND \`handler_name\` IS NOT NULL ` +
'WHERE `app_uid` = ? AND `handler_name` IS NOT NULL ' +
'GROUP BY `handler_name`',
params,
[appUid],
);
return new Map(
rows.map((row) => [
@@ -731,8 +450,7 @@ export class EventHandlerStore extends PuterStore {
/**
* How many of an owner's apps have at least one published handler — what a
* rent listener counts to tell a first publish (0→1) from a redeploy, and a
* last removal (1→0) from one of several. Reads the apps' own sets only:
* user scopes are never the owner's.
* last removal (1→0) from one of several.
*
* `createdBefore` (epoch ms) restricts to apps whose events worker existed
* before that time, by its earliest handler; used to bill only what was
@@ -764,36 +482,10 @@ export class EventHandlerStore extends PuterStore {
return Number(rows[0]?.total ?? 0);
}
/**
* How many apps a user holds a scope in — the rent count for user scopes,
* with `createdBefore` (epoch ms) meaning what it does for
* {@link countAppsWithHandlersForOwner}.
*/
async countUserScopesWithHandlersForUser(
userId: number,
opts: { createdBefore?: number } = {},
): Promise<number> {
const params: unknown[] = [userId];
let having = '';
if (opts.createdBefore !== undefined) {
having = ' HAVING MIN(`scope_since`) < ?';
params.push(Math.ceil(opts.createdBefore / 1000));
}
const rows = await this.clients.db.pread(
'SELECT COUNT(*) AS `total` FROM (' +
`SELECT \`app_uid\` FROM \`${USER_TABLE}\` ` +
`WHERE \`holder_user_id\` = ? GROUP BY \`app_uid\`${having}` +
') AS `counted`',
params,
);
return Number(rows[0]?.total ?? 0);
}
/**
* The events workers an owner is running, one row per app with at least one
* published handler. Keyset-paginated on `app_uid`, since the grouping key
* is the only stable order a set this shape can offer. Reads the apps' own
* sets only.
* is the only stable order a set this shape can offer.
*/
async listEventsWorkersForOwner(
ownerUserId: number,
@@ -852,184 +544,21 @@ export class EventHandlerStore extends PuterStore {
return result;
}
/**
* The scopes one user holds, one row per app, keyset-paginated on
* `app_uid`. `total` counts every scope the user holds (or the one app's,
* when `appUid` narrows it).
*/
async listUserScopesForUser(
userId: number,
opts: ListEventsWorkersOptions = {},
): Promise<PageResult<UserScopeSummary>> {
const limit = Math.min(
Math.max(
1,
Math.floor(opts.limit ?? EVENTS_WORKERS_LIST_DEFAULT_LIMIT),
),
EVENTS_WORKERS_LIST_LIMIT_CAP,
);
const after = decodeCursor(opts.cursor)?.appUid;
const where = [`\`${USER_TABLE}\`.\`holder_user_id\` = ?`];
const params: unknown[] = [userId];
if (typeof opts.appUid === 'string') {
where.push(`\`${USER_TABLE}\`.\`app_uid\` = ?`);
params.push(opts.appUid);
}
const totalRows = await this.clients.db.read(
'SELECT COUNT(*) AS `total` FROM (' +
`SELECT \`${USER_TABLE}\`.\`app_uid\` FROM \`${USER_TABLE}\` ` +
`WHERE ${where.join(' AND ')} GROUP BY \`${USER_TABLE}\`.\`app_uid\`` +
') AS `counted`',
params,
);
if (typeof after === 'string') {
where.push(`\`${USER_TABLE}\`.\`app_uid\` > ?`);
params.push(after);
}
const rows = await this.clients.db.read(
`SELECT \`${USER_TABLE}\`.\`app_uid\` AS \`app_uid\`, ` +
`\`${APP_TABLE}\`.\`name\` AS \`app_name\`, ` +
`\`${APP_TABLE}\`.\`title\` AS \`app_title\`, ` +
'COUNT(*) AS `handler_count`, ' +
`MIN(\`${USER_TABLE}\`.\`created_at\`) AS \`created_at\`, ` +
`MAX(\`${USER_TABLE}\`.\`updated_at\`) AS \`updated_at\` ` +
`FROM \`${USER_TABLE}\` ` +
`LEFT JOIN \`${APP_TABLE}\` ON \`${APP_TABLE}\`.\`uid\` = \`${USER_TABLE}\`.\`app_uid\` ` +
`WHERE ${where.join(' AND ')} ` +
`GROUP BY \`${USER_TABLE}\`.\`app_uid\`, \`${APP_TABLE}\`.\`name\`, \`${APP_TABLE}\`.\`title\` ` +
`ORDER BY \`${USER_TABLE}\`.\`app_uid\` LIMIT ?`,
[...params, limit + 1],
);
const page = keysetPage(rows, limit, undefined, (last) => ({
appUid: last.app_uid,
}));
const result: PageResult<UserScopeSummary> = {
items: page.rows.map((row) => ({
appUid: String(row.app_uid),
appName: row.app_name == null ? '' : String(row.app_name),
appTitle: row.app_title == null ? '' : String(row.app_title),
handlerCount: Number(row.handler_count) || 0,
createdAt: Number(row.created_at) || 0,
updatedAt: Number(row.updated_at) || 0,
})),
total: Number(totalRows[0]?.total ?? 0),
};
if (page.cursor) result.cursor = page.cursor;
return result;
}
/**
* The user scopes one app holds, ordered by holder. A batch scan: it
* carries no `total`, to spare a COUNT per page. `cursor` is null at the
* end.
*/
async listUserScopes(
appUid: string,
opts: { limit?: number; cursor?: string } = {},
): Promise<{ items: AppUserScope[]; cursor: string | null }> {
const limit = Math.min(
Math.max(1, Math.floor(opts.limit ?? USER_SCOPES_PAGE_DEFAULT)),
USER_SCOPES_PAGE_CAP,
);
const after = Number(decodeCursor(opts.cursor)?.holder);
const rows = await this.clients.db.read(
'SELECT `holder_user_id`, COUNT(*) AS `handler_count`, ' +
'MIN(`created_at`) AS `created_at`, ' +
'MAX(`updated_at`) AS `updated_at` ' +
`FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` > ? ` +
'GROUP BY `holder_user_id` ORDER BY `holder_user_id` LIMIT ?',
[appUid, Number.isFinite(after) ? after : 0, limit + 1],
);
const page = keysetPage(rows, limit, undefined, (last) => ({
holder: Number(last.holder_user_id),
}));
return {
items: page.rows.map((row) => ({
holder: Number(row.holder_user_id),
handlerCount: Number(row.handler_count) || 0,
createdAt: Number(row.created_at) || 0,
updatedAt: Number(row.updated_at) || 0,
})),
cursor: page.cursor ?? null,
};
}
/**
* One keyset page of user-scope handlers last touched before `olderThan`
* (unix seconds) that no subscription row of any state is bound to. A batch
* scan: it carries no `total`, to spare a COUNT per page. `cursor` is null
* at the end. A candidate may be bound before it is deleted, so delete
* through {@link deleteUserHandlerIfUnbound}.
*/
async listUnboundUserHandlers(
olderThan: number,
opts: { limit?: number; cursor?: string } = {},
): Promise<{ items: UnboundUserHandler[]; cursor: string | null }> {
const limit = Math.max(
1,
Math.floor(opts.limit ?? UNBOUND_HANDLERS_BATCH),
);
const after = Number(decodeCursor(opts.cursor)?.id);
const rows = await this.clients.db.read(
'SELECT h.`id`, h.`app_uid`, h.`holder_user_id`, h.`name` ' +
`FROM \`${USER_TABLE}\` h ` +
'WHERE h.`id` > ? AND h.`updated_at` < ? ' +
`AND NOT EXISTS (SELECT 1 FROM \`${SUBSCRIPTION_TABLE}\` s ` +
'WHERE s.`holder_user_id` = h.`holder_user_id` ' +
'AND s.`app_uid` = h.`app_uid` AND s.`handler_scope` = 1 ' +
'AND s.`handler_name` = h.`name`) ' +
'ORDER BY h.`id` LIMIT ?',
[Number.isFinite(after) ? after : 0, olderThan, limit + 1],
);
const page = keysetPage(rows, limit, undefined);
return {
items: page.rows.map((row) => ({
id: Number(row.id),
appUid: String(row.app_uid),
holder: Number(row.holder_user_id),
name: String(row.name),
})),
cursor: page.cursor ?? null,
};
}
// -- Internals ---------------------------------------------------
async #insert(
scope: HandlerScope,
appUid: string,
name: string,
source: string,
sourceHash: string,
broadcast: boolean,
): Promise<EventHandler> {
// A user scope's count read also yields when the scope began, so its
// age survives the handlers that held it.
let scopeSince: number | null = null;
if (isUserScope(scope)) {
const [state] = await this.clients.db.pread(
'SELECT COUNT(*) AS `total`, MIN(`scope_since`) AS `since` ' +
`FROM \`${USER_TABLE}\` WHERE \`app_uid\` = ? AND \`holder_user_id\` = ?`,
[scope.appUid, scope.holder],
);
if (Number(state?.total ?? 0) >= handlerCapOf(scope))
throw tooManyHandlers(scope);
scopeSince = state?.since == null ? null : Number(state.since);
} else if ((await this.countForApp(scope)) >= handlerCapOf(scope))
throw tooManyHandlers(scope);
if ((await this.countForApp(appUid)) >= EVENTS_HANDLERS_PER_APP)
throw tooManyHandlers();
const at = nowSeconds();
try {
await this.clients.db.insert(tableOf(scope), {
app_uid: scope.appUid,
...(isUserScope(scope)
? {
holder_user_id: scope.holder,
scope_since: scopeSince ?? at,
}
: {}),
await this.clients.db.insert(TABLE, {
app_uid: appUid,
name,
source,
source_hash: sourceHash,
@@ -1044,25 +573,9 @@ export class EventHandlerStore extends PuterStore {
if (isUniqueViolation(err)) throw publishConflict(name);
throw err;
}
// Concurrent creates can all pass the count above. Counted again with
// this row in it, each one past the cap takes itself back out — at the
// last slot that may refuse more than it had to, never fewer. User
// scopes only: reads stop at the cap, so a row past it would be unseen.
if (
isUserScope(scope) &&
(await this.countForApp(scope)) > handlerCapOf(scope)
) {
const { where, params } = scopeFilter(scope);
await this.clients.db.write(
`DELETE FROM \`${tableOf(scope)}\` WHERE ${where} AND \`name\` = ?`,
[...params, name],
);
await this.#forgetSet(scope);
throw tooManyHandlers(scope);
}
await this.#forgetSet(scope, broadcast);
await this.#forgetSet(appUid);
return {
appUid: scope.appUid,
appUid,
name,
source,
sourceHash,
@@ -1072,25 +585,23 @@ export class EventHandlerStore extends PuterStore {
}
async #update(
scope: HandlerScope,
existing: EventHandler,
source: string,
sourceHash: string,
): Promise<EventHandler> {
const at = nowSeconds();
const { where, params } = scopeFilter(scope);
// The `source_hash` predicate is what makes the check-then-write a
// compare-and-set: a publish that raced past the read above updates
// nothing here, and its caller is told.
const updated = await this.clients.db.write(
`UPDATE \`${tableOf(scope)}\` SET \`source\` = ?, \`source_hash\` = ?, ` +
`\`updated_at\` = ? WHERE ${where} AND \`name\` = ? ` +
`UPDATE \`${TABLE}\` SET \`source\` = ?, \`source_hash\` = ?, ` +
'`updated_at` = ? WHERE `app_uid` = ? AND `name` = ? ' +
'AND `source_hash` = ?',
[
source,
sourceHash,
at,
...params,
existing.appUid,
existing.name,
existing.sourceHash,
],
@@ -1098,37 +609,20 @@ export class EventHandlerStore extends PuterStore {
if (updated?.anyRowsAffected === false)
throw publishConflict(existing.name);
await this.#forgetSet(scope);
await this.#forgetSet(existing.appUid);
return { ...existing, source, sourceHash, updatedAt: at };
}
/**
* Drop the delivery path's cached set here, and on every peer if
* `broadcast`.
*/
async #forgetSet(scope: HandlerScope, broadcast = true): Promise<void> {
/** Drop the delivery path's cached set here and on every peer. */
async #forgetSet(appUid: string): Promise<void> {
await this.publishCacheKeys({
keys: [handlerSetCacheKey(scope), replicaSetCacheKey(scope)],
broadcast,
});
}
/** {@link #forgetSet} for many scopes, as one peer message. */
async #forgetSets(scopes: HandlerScope[]): Promise<void> {
await this.publishCacheKeys({
keys: scopes.flatMap((scope) => [
handlerSetCacheKey(scope),
replicaSetCacheKey(scope),
]),
keys: [handlerSetCacheKey(appUid)],
broadcast: true,
});
}
#assertName(name: unknown, scope: HandlerScope): string {
if (
typeof name !== 'string' ||
!isValidHandlerName(name, isUserScope(scope))
)
#assertName(name: unknown): string {
if (typeof name !== 'string' || !isValidHandlerName(name))
throw invalidName();
return name;
}
@@ -155,15 +155,6 @@ const liveRegionsByToken = (
return live;
};
/**
* Present on every durable row this code caches: the handler scope is stated (a
* user row also carries `handlerScope`). A node without user scopes writes rows
* without it, and its entries say nothing about scope.
*/
const SCOPE_MARKER = 'sc';
const SCOPE_LOOKUP_BATCH = 200;
/** A row we cannot read is a row we cannot deliver against. */
const parseRow = (raw: string): DispatchSubscription | null => {
try {
@@ -1024,11 +1015,7 @@ export class EventSubscriptionStore extends PuterStore {
const write = this.clients.redis.pipeline();
for (const row of rows) {
const key = tokenKey(ownerUserId, row.token);
write.hset(
key,
row.subId,
JSON.stringify({ ...row, [SCOPE_MARKER]: 1 }),
);
write.hset(key, row.subId, JSON.stringify(row));
write.expire(key, DURABLE_CACHE_TTL_SECONDS);
write.hset(durableMapKey(ownerUserId), row.subId, row.token);
}
@@ -1282,96 +1269,9 @@ export class EventSubscriptionStore extends PuterStore {
subs.push(parsed);
}
}
await this.#settleScopes(ownerUserId, subs);
return subs;
}
/**
* Rows a node without user scopes cached carry no scope marker, so a user
* row among them reads as an app row. Those with a handler are resolved
* from the table (replica, then primary for what it lacks) and rewritten
* with the marker; marked rows cost nothing.
*/
async #settleScopes(
ownerUserId: number,
rows: DispatchSubscription[],
): Promise<void> {
const unmarked: DispatchSubscription[] = [];
for (const row of rows) {
const marked = (row as unknown as Record<string, unknown>)[
SCOPE_MARKER
];
delete (row as unknown as Record<string, unknown>)[SCOPE_MARKER];
if (marked === undefined && row.durable === true && row.handlerName)
unmarked.push(row);
}
if (unmarked.length === 0) return;
// A failed lookup throws, as a table read that fails does for the
// rest of dispatch: a row of unknown scope must never be returned,
// since it would read as an app row.
const scopes = new Map<string, number>();
const lookup = async (
read: 'read' | 'pread',
ids: string[],
): Promise<void> => {
const found = await this.clients.db[read](
'SELECT `sub_id`, `handler_scope` FROM `event_subscriptions` ' +
`WHERE \`sub_id\` IN (${ids.map(() => '?').join(', ')})`,
ids,
);
for (const row of found)
scopes.set(String(row.sub_id), Number(row.handler_scope));
};
for (let i = 0; i < unmarked.length; i += SCOPE_LOOKUP_BATCH) {
const ids = unmarked
.slice(i, i + SCOPE_LOOKUP_BATCH)
.map((row) => row.subId);
await lookup('read', ids);
const missing = ids.filter((id) => !scopes.has(id));
if (missing.length > 0) await lookup('pread', missing);
}
// In neither the replica nor the primary: a ghost entry. Dropped,
// as the normal path treats a row that is gone, so it is not
// resolved again on every read.
const ghosts = unmarked.filter((row) => !scopes.has(row.subId));
if (ghosts.length > 0) {
const drop = this.clients.redis.pipeline();
for (const row of ghosts)
drop.hdel(tokenKey(ownerUserId, row.token), row.subId);
await drop.exec();
for (const row of ghosts) rows.splice(rows.indexOf(row), 1);
}
const resolved = unmarked.filter((row) => scopes.has(row.subId));
for (const row of resolved)
if (scopes.get(row.subId) === 1)
(row as DurableSubscription).handlerScope = 'user';
// Rewritten only where the entry still stands, so a row dropped
// meanwhile is not brought back. Failing to rewrite costs a
// repeat lookup, not a wrong answer.
try {
const check = this.clients.redis.pipeline();
for (const row of resolved)
check.hexists(tokenKey(ownerUserId, row.token), row.subId);
const standing = (await check.exec()) ?? [];
const write = this.clients.redis.pipeline();
resolved.forEach((row, i) => {
if (Number(standing[i]?.[1]) !== 1) return;
write.hset(
tokenKey(ownerUserId, row.token),
row.subId,
JSON.stringify({ ...row, [SCOPE_MARKER]: 1 }),
);
});
await write.exec();
} catch (err) {
console.warn('[events] could not mark cached row scopes', err);
}
}
/**
* One pipelined first page per token, which is the whole hash for any token
* small enough to be stored compactly; only a larger one is scanned
@@ -1416,7 +1316,6 @@ export class EventSubscriptionStore extends PuterStore {
}
subs.push(...taken.values());
}
await this.#settleScopes(ownerUserId, subs);
return subs;
}
@@ -326,7 +326,7 @@ describe('the notify flag', () => {
});
describe('live handles by owner and app', () => {
it("lists only the owner's live handles in that app, oldest first", async () => {
it('lists only the owner\'s live handles in that app, oldest first', async () => {
const appUid = 'app-owner-listing';
const first = await store().mint(
input({ appUid, keyPrefix: 'owned-a:' }),
@@ -502,11 +502,13 @@ describe('the live cache across nodes', () => {
expect(await store().getLive(row.handle)).toBeNull();
});
it("drops an owner's cached rows when the owner is suspended, here and on other nodes", async () => {
it('drops an owner\'s cached rows when the owner is suspended, here and on other nodes', async () => {
const row = await store().mint(
input({ appUid: 'app-forget', keyPrefix: 'forget-owner:' }),
);
expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
false,
);
const setSuspended = (value: number) =>
env.server.clients.db.write(
'UPDATE `user` SET `suspended` = ? WHERE `id` = ?',
@@ -532,7 +534,11 @@ describe('the live cache across nodes', () => {
([key]) => key === 'outer.pubsub.kv.share.forgetOwner',
),
).toEqual([
['outer.pubsub.kv.share.forgetOwner', { ownerUserId }, {}],
[
'outer.pubsub.kv.share.forgetOwner',
{ ownerUserId },
{},
],
]);
// Another node lifting the suspension reaches this one's cache.
@@ -551,11 +557,13 @@ describe('the live cache across nodes', () => {
}
});
it("drops an owner's cached rows when the user store suspends or unsuspends them", async () => {
it('drops an owner\'s cached rows when the user store suspends or unsuspends them', async () => {
const row = await store().mint(
input({ appUid: 'app-forget', keyPrefix: 'forget-store:' }),
);
expect((await store().getLive(row.handle))?.ownerSuspended).toBe(false);
expect((await store().getLive(row.handle))?.ownerSuspended).toBe(
false,
);
try {
await env.server.stores.user.update(ownerUserId, {
suspended: true,
@@ -887,27 +895,21 @@ describe('retired handle retention', () => {
expect(await exists(live)).toBe(true);
});
it('purges retired handles past the retention, keeping live and recent rows', async () => {
it('purges once per day across nodes, keeping live and recent rows', async () => {
const kvShare = env.server.services.kvShare;
await env.server.clients.redis.del('kvShare:purge:gate');
const old = await retiredAt('purge-old:', now() - 31 * DAY);
const recent = await retiredAt('purge-recent:', now() - 29 * DAY);
const live = await retiredAt('purge-live:', null);
expect(
await kvShare.purgeRevokedHandles(new AbortController().signal),
).toBeGreaterThanOrEqual(1);
expect(await kvShare.purgeRevokedHandles()).toBeGreaterThanOrEqual(1);
expect(await exists(old)).toBe(false);
expect(await exists(recent)).toBe(true);
expect(await exists(live)).toBe(true);
});
it('stops purging between batches once its signal aborts', async () => {
const kvShare = env.server.services.kvShare;
const old = await retiredAt('purge-aborted:', now() - 31 * DAY);
const controller = new AbortController();
controller.abort();
expect(await kvShare.purgeRevokedHandles(controller.signal)).toBe(0);
expect(await exists(old)).toBe(true);
// Another node checking the same day finds the gate held.
const again = await retiredAt('purge-again:', now() - 31 * DAY);
expect(await kvShare.purgeRevokedHandles()).toBeNull();
expect(await exists(again)).toBe(true);
});
});
@@ -18,10 +18,7 @@
*/
import { randomUUID } from 'node:crypto';
import {
EVENTS_INVOKE_TIMEOUT_MS,
EVENTS_WORKER_LEASE_FACTOR,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import { EVENTS_INVOKE_TIMEOUT_MS } from '../../clients/events/EventsWorkerInvokerClient.js';
import {
EVENTS_FAILURE_COUNTER_TTL_MS,
EVENTS_PENDING_DELIVERIES_PER_SUBSCRIPTION,
@@ -319,7 +316,7 @@ export class PendingDeliveryStore extends PuterStore {
typeof configured === 'number' && configured > 0
? configured
: EVENTS_INVOKE_TIMEOUT_MS;
return invokeTimeoutMs * EVENTS_WORKER_LEASE_FACTOR;
return invokeTimeoutMs * 2;
}
// -- Writes ------------------------------------------------------
+2 -5
View File
@@ -19,11 +19,8 @@
import { HttpError } from '../../core/http/HttpError.js';
/**
* How wide one column is, and the name a caller knows the value by. Called by
* extensions.
*/
export interface ColumnWidth {
/** How wide one column is, and the name a caller knows the value by. */
interface ColumnWidth {
/** Column in the migrations, so a schema change is greppable from here. */
column: string;
max: number;
-22
View File
@@ -29,12 +29,6 @@ import type { FsOp } from '../../services/events/subjects.js';
* is declared once here rather than per store.
*/
/**
* Which handler set a subscription's `handlerName` refers to: the app's own
* (`'app'`) or the holder's scope of that app (`'user'`). Stored as 0 and 1.
*/
export type HandlerScopeKind = 'app' | 'user';
/**
* Whether a row's events also run the app's published handler in its events
* worker. Only an app has a worker, and only a named handler can be run.
@@ -44,20 +38,6 @@ export const runsInWorker = (row: {
handlerName?: string | null;
}): boolean => row.appUid !== null && !!row.handlerName;
/**
* Token, and subject, of the subscription row a recurring or `once` schedule
* owns. Those rows carry the schedule's lifecycle (suspend, resume, reap) and
* are kept out of dispatch, the durable quotas and the durable listing. Called
* by extensions.
*/
export const SCHEDULE_TOKEN_PREFIX = 'schedule:';
export const scheduleToken = (scheduleUid: string): string =>
`${SCHEDULE_TOKEN_PREFIX}${scheduleUid}`;
export const isScheduleRow = (row: { token: string }): boolean =>
row.token.startsWith(SCHEDULE_TOKEN_PREFIX);
/**
* What a row's delivery re-check runs against. An access mode for a filesystem
* row, which composes with its anchor uid; the whole grant string for a row on
@@ -103,8 +83,6 @@ export interface DispatchSubscription {
durable?: true;
delivery?: DeliveryClass;
handlerName?: string | null;
/** Durable rows only: absent means `'app'`. */
handlerScope?: HandlerScopeKind;
/** Durable rows only: handed to the handler, and read nowhere else. */
context?: string | null;
}
@@ -33,9 +33,6 @@ import { PuterStore } from '../types';
* }} NotificationInsert
*/
/** How long a probe that found no `send_key` column stands. */
const SEND_KEY_REPROBE_MS = 60_000;
/**
* Fixed width and compared byte for byte on every engine, whatever `group`
* holds.
@@ -78,15 +75,6 @@ const sliceWhere = (userId, { audience, appUid, excludeTypes }) => {
};
export class NotificationStore extends PuterStore {
#sendKeyPresent = false;
#sendKeyProbedAt = Number.NEGATIVE_INFINITY;
/** @type {Promise<void> | null} */
#sendKeyProbe = null;
async onServerStart() {
await this.#hasSendKey();
}
// -- Reads --------------------------------------------------------
/**
@@ -108,20 +96,6 @@ export class NotificationStore extends PuterStore {
return this.#normalizeRow(rows[0]) ?? null;
}
/**
* The row a delayed send wrote or last replaced, from the primary. Called
* by extensions.
*
* @param {string} sendKey
*/
async getBySendKey(sendKey) {
const rows = await this.clients.db.pread(
'SELECT * FROM `notification` WHERE `send_key` = ? LIMIT 1',
[sendKey],
);
return this.#normalizeRow(rows[0]) ?? null;
}
/**
* The newest row `appUid` wrote for the user under `group`, or null when it
* was shown or acknowledged: an older row is never picked up in its place.
@@ -315,57 +289,6 @@ export class NotificationStore extends PuterStore {
return rows.map((r) => this.#normalizeRow(r));
}
/**
* The newest row id in a mailbox slice, or null when it has none, from the
* primary. Scoped like `listScoped`; rows of every type count.
*
* @param {number} userId
* @param {{ audience: string; appUid?: string | null }} scope
* @returns {Promise<number | null>}
*/
async latestScopedId(userId, { audience, appUid }) {
const { where, params } = sliceWhere(userId, {
audience,
appUid,
excludeTypes: [],
});
const [row] = await this.clients.db.pread(
`SELECT \`id\` FROM \`notification\`
WHERE ${where.join(' AND ')}
ORDER BY \`id\` DESC
LIMIT 1`,
params,
);
return row ? Number(row.id) : null;
}
/**
* The latest replacement in a mailbox slice by (`replaced_at`, `id`), or
* null when none was replaced, from the primary. Scoped like `listScoped`.
*
* @param {number} userId
* @param {{ audience: string; appUid?: string | null }} scope
* @returns {Promise<{ replacedAt: number; id: number } | null>}
*/
async latestScopedReplacement(userId, { audience, appUid }) {
const { where, params } = sliceWhere(userId, {
audience,
appUid,
excludeTypes: [],
});
where.push('`replaced_at` IS NOT NULL');
const [row] = await this.clients.db.pread(
`SELECT \`replaced_at\`, \`id\` FROM \`notification\`
WHERE ${where.join(' AND ')}
ORDER BY \`replaced_at\` DESC, \`id\` DESC
LIMIT 1`,
params,
);
return row
? { replacedAt: Number(row.replaced_at), id: Number(row.id) }
: null;
}
// -- Writes -------------------------------------------------------
/**
@@ -418,7 +341,6 @@ export class NotificationStore extends PuterStore {
type,
group_key: key,
replaced_at: null,
send_key: null,
});
}
@@ -432,95 +354,6 @@ export class NotificationStore extends PuterStore {
return (await this.create(args)).uid;
}
/**
* The statements that write a delayed send's notification, for the caller's
* transaction. Each runs only while `guard` holds. With `replaceUid` the
* open row is replaced in place and takes `sendKey`; the insert, whose uid
* is `sendKey`, then runs only if that row was shown or dismissed in
* between. Called by extensions.
*
* @param {{
* sendKey: string;
* userId: number;
* value: unknown;
* type: string;
* audience: string;
* appUid: string | null;
* group?: string | null;
* replaceUid?: string | null;
* at: number;
* }} send
* `at` is epoch ms, the row's time either way.
* @param {{ table: string; where: string; values: unknown[] }} guard
* @returns {{ statement: string; values: unknown[] }[]}
*/
sendEntries(send, guard) {
const db = this.clients.db;
const serialized = JSON.stringify(send.value ?? {});
const seconds = Math.floor(send.at / 1000);
const fence = `SELECT 1 FROM \`${guard.table}\` WHERE ${guard.where}`;
const entries = [];
if (send.replaceUid) {
entries.push({
statement:
'UPDATE `notification` SET `value` = ?, `shown` = NULL, ' +
`\`created_at\` = ${db.timestampFromUnixSql()}, ` +
'`replaced_at` = ?, `send_key` = ? WHERE `uid` = ? ' +
'AND `user_id` = ? AND `shown` IS NULL ' +
`AND \`acknowledged\` IS NULL AND EXISTS (${fence})`,
values: [
serialized,
seconds,
send.at,
send.sendKey,
send.replaceUid,
send.userId,
...guard.values,
],
});
}
// Already written: the new row's uid is the key, or a replaced row
// carries it. Point lookups on `uid`, since a miss on the mostly-NULL
// `send_key` index gap-locks where every plain insert goes.
const written = send.replaceUid
? {
sql:
'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?) ' +
'AND NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ? AND `send_key` = ?)',
values: [send.sendKey, send.replaceUid, send.sendKey],
}
: {
sql: 'NOT EXISTS (SELECT 1 FROM `notification` WHERE `uid` = ?)',
values: [send.sendKey],
};
// Postgres can't type a bare parameter in a SELECT list.
const param = (type) =>
db.case({ postgres: `CAST(? AS ${type})`, otherwise: '?' });
entries.push({
statement:
'INSERT INTO `notification` (`uid`, `user_id`, `value`, `type`, ' +
'`audience`, `app_uid`, `group_key`, `created_at`, `send_key`) ' +
`SELECT ${param('TEXT')}, ${param('INTEGER')}, ${param('JSONB')}, ` +
`${param('TEXT')}, ${param('TEXT')}, ${param('TEXT')}, ` +
`${param('TEXT')}, ${db.timestampFromUnixSql()}, ${param('TEXT')} ` +
`FROM \`${guard.table}\` WHERE ${guard.where} AND ${written.sql}`,
values: [
send.sendKey,
send.userId,
serialized,
send.type,
send.audience,
send.appUid,
groupKey(send.group),
seconds,
send.sendKey,
...guard.values,
...written.values,
],
});
return entries;
}
/**
* Rewrite a notification the recipient hasn't dismissed. `shown` is cleared
* with it, so changed wording goes out again on their next connect —
@@ -529,10 +362,6 @@ export class NotificationStore extends PuterStore {
* `onlyUnseen` also skips a row already shown. `replacedAt`, in epoch ms,
* records the rewrite as a replacement and dates the row to it.
*
* `send_key` is cleared with it whenever the column exists, on every node:
* the content is no longer what a delayed send wrote, so its recovery must
* not deliver it under that send's key.
*
* False when no such row exists, so callers can fall back to a fresh
* notification instead of dropping what they were reporting.
*
@@ -549,7 +378,6 @@ export class NotificationStore extends PuterStore {
const serialized =
typeof value === 'string' ? value : JSON.stringify(value ?? {});
const set = ['`value` = ?', '`shown` = NULL'];
if (await this.#hasSendKey()) set.push('`send_key` = NULL');
const params = [serialized];
if (replacedAt !== undefined) {
set.push(
@@ -650,45 +478,6 @@ export class NotificationStore extends PuterStore {
// -- Internals ----------------------------------------------------
/**
* Whether `notification.send_key` exists, asked of the primary. Present is
* kept for the process; absent is asked again at most once a minute, so a
* node started before the column was added clears it soon after.
*
* @returns {Promise<boolean>}
*/
async #hasSendKey() {
if (this.#sendKeyPresent) return true;
if (Date.now() - this.#sendKeyProbedAt < SEND_KEY_REPROBE_MS)
return false;
if (this.#sendKeyProbe) {
await this.#sendKeyProbe;
return this.#sendKeyPresent;
}
const probe = this.#probeSendKey();
this.#sendKeyProbe = probe;
try {
await probe;
} finally {
if (this.#sendKeyProbe === probe) this.#sendKeyProbe = null;
}
return this.#sendKeyPresent;
}
/** Never rejects: an error reads as no column. */
async #probeSendKey() {
try {
await this.clients.db.pread(
'SELECT `send_key` FROM `notification` WHERE 1 = 0',
[],
);
this.#sendKeyPresent = true;
} catch {
this.#sendKeyPresent = false;
}
this.#sendKeyProbedAt = Date.now();
}
#normalizeRow(row) {
if (!row) return null;
if (typeof row.value === 'string') {
@@ -1,228 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* `send_key` is cleared on every rewrite once the column exists, whichever node
* rewrites and whatever its config: a row a delayed send wrote and another node
* replaced must not be delivered again under that send's key.
*/
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
it,
vi,
} from 'vitest';
import type { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import { NotificationStore } from './NotificationStore.js';
let server: PuterServer;
let userId: number;
/** A store on the same database, as another node with its own config. */
const node = (config: Partial<IConfig>): NotificationStore =>
new NotificationStore(
config as IConfig,
server.clients as never,
server.stores as never,
);
/** A row as a delayed send writes it, keyed `sendKey`. */
const writeSent = async (store: NotificationStore): Promise<string> => {
const sendKey = uuidv4();
await server.clients.db.batchWrite(
store.sendEntries(
{
sendKey,
userId,
value: { title: 'later' },
type: 'app-message',
audience: 'account',
appUid: null,
at: Date.now(),
},
{ table: 'user', where: '`id` = ?', values: [userId] },
),
);
return sendKey;
};
const sendKeyOf = async (uid: string): Promise<unknown> => {
const [row] = await server.clients.db.pread(
'SELECT `send_key` FROM `notification` WHERE `uid` = ?',
[uid],
);
return row?.send_key;
};
beforeAll(async () => {
server = await setupTestServer();
const username = `sendkey-${Math.random().toString(36).slice(2, 10)}`;
const user = await server.stores.user.create({
username,
uuid: uuidv4(),
password: null,
email: `${username}@test.local`,
});
userId = user.id;
});
afterAll(async () => {
await server?.shutdown();
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('clearing send_key on a rewrite', () => {
it('clears it from a row another node wrote, whatever either config says', async () => {
const writer = node({ events: { enabled: true } } as Partial<IConfig>);
const rewriter = node({});
const sendKey = await writeSent(writer);
expect(await sendKeyOf(sendKey)).toBe(sendKey);
await expect(
rewriter.updateValue(sendKey, userId, { title: 'replaced' }),
).resolves.toBe(true);
expect(await sendKeyOf(sendKey)).toBeNull();
});
it('probes once and keeps a present column for the process', async () => {
const store = node({});
const pread = vi.spyOn(server.clients.db, 'pread');
const first = await writeSent(store);
const second = await writeSent(store);
await store.updateValue(first, userId, { v: 1 });
await store.updateValue(second, userId, { v: 2 });
const probes = pread.mock.calls.filter(([sql]) =>
String(sql).includes('WHERE 1 = 0'),
);
expect(probes).toHaveLength(1);
expect(await sendKeyOf(first)).toBeNull();
expect(await sendKeyOf(second)).toBeNull();
});
it('rewrites without naming the column while it is absent, and clears it after the next probe once it exists', async () => {
const db = server.clients.db;
const store = node({});
const sendKey = await writeSent(store);
const write = vi.spyOn(db, 'write');
const realPread = db.pread.bind(db);
let columnExists = false;
vi.spyOn(db, 'pread').mockImplementation(async (sql, values) => {
if (!columnExists && String(sql).includes('WHERE 1 = 0'))
throw new Error('no such column: send_key');
return realPread(sql, values);
});
let now = Date.now();
vi.spyOn(Date, 'now').mockImplementation(() => now);
const rewriteSql = () =>
String(
write.mock.calls.findLast(([sql]) =>
String(sql).startsWith('UPDATE `notification` SET `value`'),
)?.[0],
);
await store.updateValue(sendKey, userId, { v: 1 });
expect(rewriteSql()).not.toContain('send_key');
expect(await sendKeyOf(sendKey)).toBe(sendKey);
// Within the minute the absent answer stands.
columnExists = true;
now += 30_000;
await store.updateValue(sendKey, userId, { v: 2 });
expect(rewriteSql()).not.toContain('send_key');
expect(await sendKeyOf(sendKey)).toBe(sendKey);
now += 31_000;
await store.updateValue(sendKey, userId, { v: 3 });
expect(rewriteSql()).toContain('`send_key` = NULL');
expect(await sendKeyOf(sendKey)).toBeNull();
});
it('shares one probe between rewrites that race it', async () => {
const db = server.clients.db;
const store = node({});
const keys = await Promise.all([writeSent(store), writeSent(store)]);
const pread = vi.spyOn(db, 'pread');
await Promise.all(
keys.map((uid) => store.updateValue(uid, userId, { v: 1 })),
);
const probes = pread.mock.calls.filter(([sql]) =>
String(sql).includes('WHERE 1 = 0'),
);
expect(probes).toHaveLength(1);
for (const uid of keys) expect(await sendKeyOf(uid)).toBeNull();
});
});
describe('writing a delayed send twice', () => {
const entriesFor = (sendKey: string, replaceUid: string | null = null) =>
node({}).sendEntries(
{
sendKey,
userId,
value: { title: 'later' },
type: 'app-message',
audience: 'account',
appUid: null,
replaceUid,
at: Date.now(),
},
{ table: 'user', where: '`id` = ?', values: [userId] },
);
const rowsKeyed = async (uid: string): Promise<number> =>
(
await server.clients.db.pread(
'SELECT 1 FROM `notification` WHERE `uid` = ?',
[uid],
)
).length;
it('writes nothing again once the row was rewritten and lost its key', async () => {
const sendKey = await writeSent(node({}));
await node({}).updateValue(sendKey, userId, { title: 'replaced' });
expect(await sendKeyOf(sendKey)).toBeNull();
await server.clients.db.batchWrite(entriesFor(sendKey));
expect(await rowsKeyed(sendKey)).toBe(1);
});
it('adds no row once the replaced row carries the key', async () => {
const replaced = await writeSent(node({}));
const sendKey = uuidv4();
await server.clients.db.batchWrite(entriesFor(sendKey, replaced));
expect(await sendKeyOf(replaced)).toBe(sendKey);
expect(await rowsKeyed(sendKey)).toBe(0);
await server.clients.db.batchWrite(entriesFor(sendKey, replaced));
expect(await rowsKeyed(sendKey)).toBe(0);
});
});
Loaded 100 of 274 files, more files were not shown because too many files have changed in this diff. Show more