Compare commits

..
Author SHA1 Message Date
404oops 465d9b35d0 fix(ai): accept larger Anthropic message requests 2026-10-30 18:56:52 +01:00
559 changed files with 8720 additions and 67663 deletions

No files matched your search

@@ -47,13 +47,3 @@ jobs:
PUTER_TEST_AI_MOONSHOT_API_KEY: ${{ secrets.PUTER_TEST_AI_MOONSHOT_API_KEY }}
PUTER_TEST_AI_ZAI_API_KEY: ${{ secrets.PUTER_TEST_AI_ZAI_API_KEY }}
PUTER_TEST_AI_ELEVENLABS_API_KEY: ${{ secrets.PUTER_TEST_AI_ELEVENLABS_API_KEY }}
# Realtime conversations through the SDK against live OpenAI. Builds
# the SDK bundle these tests load itself, so it runs even when the
# step above fails; the tests skip themselves without the key.
- name: Run live realtime conversation tests
if: ${{ !cancelled() }}
run: npm run setupExtensions && (cd src/puter-js && npm run build) && npx vitest run --config src/puter-js/tests/api/vitest.config.ts src/puter-js/tests/api/realtime/convoLive.test.ts
env:
CI: 'true'
PUTER_TEST_AI_OPENAI_API_KEY: ${{ secrets.PUTER_TEST_AI_OPENAI_API_KEY }}
-48
View File
@@ -1,48 +0,0 @@
name: Notify HeyPuter (dev)
on:
workflow_dispatch:
inputs:
puter_ref:
description: "puter branch, tag or SHA"
type: string
default: "main"
permissions:
contents: read
jobs:
authorize:
if: github.repository == 'HeyPuter/puter'
runs-on: ubuntu-latest
steps:
- name: Require maintain or admin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ACTOR: ${{ github.triggering_actor }}
run: |
ROLE=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$ACTOR/permission" --jq .role_name)
if [ "$ROLE" != "maintain" ] && [ "$ROLE" != "admin" ]; then
echo "::error::$ACTOR has role '$ROLE'; maintain or admin required"
exit 1
fi
notify:
needs: authorize
runs-on: ubuntu-latest
# Holds HEYPUTER_DISPATCH_TOKEN; limited to main so branch copies of this file can't read it.
environment: heyputer-dispatch
steps:
- name: Trigger heyputer dev deploy
env:
PUTER_REF: ${{ inputs.puter_ref }}
TOKEN: ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}
run: |
jq -nc --arg ref "$PUTER_REF" \
'{event_type: "puter-dev-requested", client_payload: {puter_ref: $ref}}' |
curl -fsS -X POST \
-H "Authorization: token $TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-d @-
echo "Requested \`$PUTER_REF\`" >> "$GITHUB_STEP_SUMMARY"
+1 -2
View File
@@ -8,7 +8,6 @@ on:
jobs:
notify:
runs-on: ubuntu-latest
environment: heyputer-dispatch
steps:
- name: Trigger heyputer build
run: |
@@ -16,4 +15,4 @@ jobs:
-H "Authorization: token ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/HeyPuter/heyputer/dispatches \
-d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
-d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
+1 -5
View File
@@ -118,13 +118,9 @@ jobs:
fail-fast: false
matrix:
include:
# The base branch lives in this repo; for a fork PR, the head
# repo's copy of it can be arbitrarily stale.
- ref: ${{ github.base_ref }}
repository: ${{ github.repository }}
artifact: base
- ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
artifact: pr
steps:
@@ -133,7 +129,7 @@ jobs:
uses: actions/checkout@v7
with:
ref: ${{ matrix.ref }}
repository: ${{ matrix.repository }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Set up Node.js
if: needs.changes.outputs.puterjs == 'true'
+1 -28
View File
@@ -161,38 +161,11 @@ An overview page answers "what is this API and is it what I need?", and a functi
When you add a page, add it to [src/docs/src/sidebar.js](src/docs/src/sidebar.js) under its area, or it won't show up in the navigation. When you add a runnable example, add its HTML to `src/docs/src/playground/examples/<slug>.html`, register the slug in [src/docs/src/examples.js](src/docs/src/examples.js), and link the code block to it by putting the slug after the language with a semicolon (` ```html;kv-set `).
### Function page layout
A function page opens with one sentence that says what the function does, in its own paragraph, such as "Given a prompt, generate an image using AI." Put any more detail in the paragraphs after it.
After the intro, a function page has these `##` sections, in this order:
1. Syntax
2. Parameters
3. Return value
4. Errors, if the function has any
5. Sections for specific topics, if the page needs them, with `## Model-specific features` last
6. Examples
[chat.md](src/docs/src/AI/chat.md) follows this layout.
- **Options as a table.** Document an options object as a table with Option, Type, and Description columns. Give each option its own row, even when two options go together, such as `width` and `height`. Order the rows by how often developers use them, most common first, without group rows. In chat.md that's the core options, then reasoning, tools, response handling, request control, and output.
- **Common options in the table, model-specific behavior in its own section.** A page whose function works with many models or providers, such as [chat.md](src/docs/src/AI/chat.md) and [txt2img.md](src/docs/src/AI/txt2img.md), splits its options this way:
- **The options table** under Parameters lists the options that work with every model, and describes each one the way it works for every model.
- **`## Model-specific features`**, the last section before Examples, covers everything else. It has a `###` for each model family or provider, named after it, such as "OpenAI models" or "Cloudflare models". Under it go the options only some models accept, and the common options a model handles differently, such as a narrower range, other allowed values, or an option it ignores.
- **Write there only what's different.** A `###` lists an option only when that model or provider adds it or handles it differently. An option that works the same everywhere stays out, even if the provider supports it, and a model or provider with nothing different gets no `###`.
- **Model-specific features is the only place for model-specific details.** Every limit, format, allowed value, default, price, or quirk that applies to some models goes there, in that model's or provider's `###`, and nowhere else. The options table and topic sections (such as Image inputs or Video input) describe what's true for every model and link to Model-specific features for the rest. Don't add a table or sentence there that compares providers or repeats a model's limits. Each copy is another source of truth to keep in sync, and copies drift when a model changes.
- **Link the two.** Directly under the options table, put `Some models accept additional options or handle these options differently. See [Model-specific features](#model-specific-features).` A row whose option behaves differently on some models can also end with `See [Model-specific features](#model-specific-features).`
- **A long topic gets its own `##` section.** When explaining an option or feature takes more than a sentence or two, such as function calling or output dimensions, give it a section between Errors and Examples and link to it from the options table. This keeps the reference sections short.
- **Usage patterns are examples.** A way of using the API, such as stopping a stream, goes in a runnable example that the relevant section links to ("See the [stop a stream example](/playground/ai-chat-stop-stream/)."), not in a section of its own.
- **Each example has a `###` heading** under `## Examples`, so every example shows in the table of contents.
### Writing docs
These apply to reference pages and recipes alike.
- **Document the developer-facing side only.** The reader is building on top of puter.js, so cover what affects their code: signatures, defaults, return shapes, errors, limits, and behavior they can observe. Leave out internal workings (which service handles the call, storage layout, caching, retries) unless they change what the developer has to do. [rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) is the exception: it publishes every limit, including timeouts and caps that come from how the backend works.
- **Document Puter as it runs on puter.com.** The docs are for developers building on production Puter. Leave out behavior that only happens on a self-hosted or differently configured server, such as a provider that isn't set up there.
- **Describe the current state.** Docs aren't a changelog. Write how the API behaves now, without "now", "no longer", "existing code is unaffected", or other wording that compares it to an older version.
- **Verify before you document.** Run the calls you describe against a running Puter instead of inferring behavior from the source.
- **Write plainly.** Use the literal words a developer would say out loud, with contractions and everyday words. No idioms or figurative verbs ("walk away from something shared with you" → "remove your own access", "nothing to page through" → "without pagination"). No formal register ("present only while more pages remain" → "includes a `cursor` if there are more pages to load"). Prefer dev terms such as atomic, concurrent, and insertion order.
@@ -228,6 +201,6 @@ Recipes are tested, copy-pasteable patterns for a task, one layer above the per-
- **Reuse existing UI primitives before writing new ones.** Windows and dialogs are `UIWindow*` functions in [src/gui/src/UI/](src/gui/src/UI/); generic pieces already exist (`UIAlert`, `UIPrompt`, `UIContextMenu`, `UINotification`, `UIPopover`, widgets in [UI/Components/](src/gui/src/UI/Components/)). A new window should read like its neighbors: an async function taking an options object, composing an HTML string, wiring behavior with jQuery, delegating to `UIWindow(...)`.
- **Match the visual language.** Use existing CSS classes (`button`, `button-primary`, window chrome, form styles) and copy the layout patterns of neighboring windows; new styles go in [src/gui/src/css/](src/gui/src/css/) following existing conventions. Verify anything positional (menus, overlays, z-index) on both desktop and mobile viewports.
- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js).
- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js). Run `npm run check-translations` before opening the PR.
- **Shared logic goes in helpers/services.** Reusable non-UI logic belongs in [src/gui/src/helpers/](src/gui/src/helpers/) or [src/gui/src/services/](src/gui/src/services/), not copy-pasted between windows.
- **Tests.** Vitest is wired for the GUI (`src/**/*.test.js`; see [appOrder.test.js](src/gui/src/UI/Dashboard/appOrder.test.js) for the shape). Extract pure logic into functions and test those. Desktop behavior driven through puter.js (`puter.ui.*`) is covered by the Playwright harness in [src/puter-js/tests/e2e/](src/puter-js/tests/e2e/) — add a spec there when you change how the desktop renders SDK-driven UI.
+1 -7
View File
@@ -73,12 +73,6 @@ COPY --from=build --chown=node:node /opt/puter .
RUN mkdir -p /etc/puter /var/puter && \
chown -R node:node /etc/puter /var/puter
# Fix outdated HOME_URL in /etc/os-release (see issue #4329)
# The base image may contain a reference to the old phoenix repository.
RUN if [ -f /etc/os-release ]; then \
sed -i 's|HOME_URL="https://github.com/HeyPuter/phoenix"|HOME_URL="https://github.com/HeyPuter/puter"|' /etc/os-release; \
fi
# Self-hosters mount their override at this exact path. The v2 loader
# deep-merges it over config.default.json (see backend/index.ts).
ENV PUTER_CONFIG_PATH=/etc/puter/config.json
@@ -93,4 +87,4 @@ USER node
HEALTHCHECK --interval=30s --timeout=3s --start-period=30s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://puter.localhost:4100/test || exit 1
CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
-7
View File
@@ -59,13 +59,6 @@ store:
row — and the query seeks past it (`WHERE (col, id) > (?, ?) ORDER BY col,
id`). Any SQL list gaining a cursor must have a deterministic `ORDER BY`
ending in a unique tiebreaker (`id`).
- The notification catch-up (`GET /events/fetch`) holds two keysets in one
sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
among rows up to that id. Each is its own indexed read, merged into one
page, so a row replaced after a client read it comes back from that cursor.
`from=end` with no cursor answers an empty page whose cursor holds the
slice's newest `id` and the slice's latest `(replaced_at, id)` as the
replacement mark.
Cursors that carry a sort also pin it: a request that passes a cursor plus a
conflicting sort is rejected with 400.
+2 -13
View File
@@ -18,17 +18,6 @@ Core subscribes the browser to a push service only when the server also publishe
When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so the page's push client registers the new subscription.
## Receipts
Before a notification is pushed, the server asks the open clients whether they received it live. It sends `notif.receipt` with `{ uid }` on the user's sockets for the app that sent it, or on the desktop's sockets for Puter's own notifications. The SDK remembers the uids its page received through `onNotification()`, `onPush()` or the desktop's feed, and answers once its page has that notification, or with `{ received: false }` when it never gets it. The server waits up to 5 seconds (`notificationAckTimeoutMs`).
An answer carries `visibilityState`, `focused` and `showsOwn`, the last being whether the page shows the notification itself. A website page shows it itself when it is hidden or unfocused, has an `onPush()` handler and notification permission, and no other tab of the same site is focused. Inside the desktop, the desktop shows it, and an app's `onPush()` result is never shown as a browser notification.
- An answer from a page that is visible and focused, or that shows the notification itself, stops the push.
- Otherwise the notification is pushed. A website's pushes go only to the browsers that subscribed through its own worker, never to Puter's `/sw.js`. An app inside the desktop pushes through `/sw.js`, which shows the app's name and icon.
- A hidden desktop tab shows no banner, so answers only from hidden desktop tabs don't stop a push of Puter's own notifications.
- A client that can't answer, such as an older puter.js, holds the push back while it is connected.
## Registering
The registry is in [src/gui/src/helpers/pushClient.js](../src/gui/src/helpers/pushClient.js). A service script registers its client like this:
@@ -68,8 +57,8 @@ interface PushClient {
}
```
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. The built-in client runs `puter.notifications.enable()`, which subscribes with the `/sw.js` registration and registers the browser through the Web Push flow above (`platform: 'webpush'`). It then resolves the `uid` of this browser's `webpush` device from `listDevices()`. It rejects when permission is refused or the browser wasn't registered.
- `disable()` runs `puter.notifications.disable()`, which sends `DELETE /push/web/subscriptions { endpoint }` and ends the browser's subscription. The tab calls it for Turn off, and for Remove on the device marked as this browser. Remove on any other device is `DELETE /push/devices/:uid`.
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. It subscribes with the `/sw.js` registration, registers the device with `POST /push/devices` (`platform: 'web'`), and resolves the device `uid` the server returned. It rejects when permission is refused or the device couldn't be registered.
- `disable()` unsubscribes and removes this browser's device with `DELETE /push/devices/:uid`. The tab calls it for Turn off, and for Remove on the device marked as this browser; Remove on any other device is a plain `DELETE`.
- `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user, and `deviceUid` is that device's `uid`.
After `enable()` resolves, the tab calls `POST /push/devices/:uid/test` and asks the user whether a notification showed up.
+1 -19
View File
@@ -12,17 +12,7 @@ Generates secrets, writes `.env` + `puter/config/config.json`, downloads `docker
## Requirements
- **Docker** with the `compose` plugin. This is the one thing you have to install
yourself — the installer checks for it and stops with an explanation rather
than installing it for you.
- **macOS / Windows:** [Docker Desktop](https://docs.docker.com/get-docker/).
Both ship the `compose` plugin.
- **Linux:** [Docker Engine](https://docs.docker.com/engine/install/) plus the
[`docker-compose-plugin`](https://docs.docker.com/compose/install/) package.
- Confirm before running anything else: `docker compose version` should print a
version, not `command not found`.
- If Docker is installed but the shell can't see it, make sure your user is in
the `docker` group (Linux) or that Docker Desktop is actually running.
- **Docker** with the `compose` plugin.
- A **domain** with DNS access — you need a wildcard record (`*.your-domain.com` → server IP). Puter routes by subdomain (`api.<domain>`, `site.<domain>`, `app.<domain>`).
- Optional: **TLS certs** (or `certbot` to grab them — see Step 3).
@@ -667,14 +657,6 @@ Migrations re-apply idempotently across pulls. Volumes are preserved.
## Troubleshooting
**`missing required command: docker` — the installer stops immediately.**
Docker isn't on your `PATH` yet. Puter self-hosting is a Docker Compose stack, so
nothing can run without it. Install [Docker Desktop](https://docs.docker.com/get-docker/)
on macOS/Windows or [Docker Engine](https://docs.docker.com/engine/install/) plus
`docker-compose-plugin` on Linux, then confirm `docker compose version` works in
the same shell and re-run the installer. On Linux, a fresh Docker install usually
needs a new login session before your user picks up the `docker` group.
**Site loads but I get a 502 / "Bad Gateway" from Caddy.**
The puter container failed to come up. `docker compose logs puter` will tell you which dependency rejected it (most often DB password mismatch between `.env` and `config.json`).
+1 -21
View File
@@ -53,29 +53,9 @@ need() {
command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
}
check_docker() {
if command -v docker >/dev/null 2>&1; then
return 0
fi
echo >&2 ""
echo >&2 "[puter-install] missing required command: docker"
echo >&2 ""
echo >&2 "Docker is required to run Puter self-hosted."
echo >&2 ""
echo >&2 "Installation instructions:"
echo >&2 " - macOS (Apple Silicon & Intel): https://docs.docker.com/desktop/setup/install/mac-install/"
echo >&2 " (Docker Desktop is recommended. Alternatives: OrbStack, Colima)"
echo >&2 " - Linux: https://docs.docker.com/engine/install/"
echo >&2 " - Windows: https://docs.docker.com/desktop/setup/install/windows-install/"
echo >&2 ""
echo >&2 "After installing Docker, make sure it's running and try again."
echo >&2 ""
exit 1
}
# ── Step 1: dependency check ────────────────────────────────────────
log "checking dependencies"
check_docker
need docker
need curl
need openssl
docker compose version >/dev/null 2>&1 \
+763 -781
View File
File diff suppressed because it is too large. Load diff
+2 -2
View File
@@ -54,6 +54,7 @@
"dev": "npm start",
"build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
"build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build",
"check-translations": "node tools/check-translations.js",
"prepare": "husky",
"build:ts": "tsc -p tsconfig.build.json && node ./tools/write-dist-package-json.mjs",
"check:puterjs:types": "node tools/checkPuterjsTypes.mjs",
@@ -78,8 +79,7 @@
"javascript-time-ago": "^2.5.11",
"libphonenumber-js": "1.13.15",
"miniflare": "^4.20260617.1",
"open": "^10.1.0",
"sharp": "^0.35.5"
"open": "^10.1.0"
},
"engines": {
"node": ">=24.0.0"
@@ -275,28 +275,6 @@ describe('AbstractDatabaseClient — SQL generation', () => {
expect(postgres().timestampFromUnixSql()).toBe('to_timestamp(?)');
});
it('reads a timestamp column back as unix seconds in each engine', () => {
expect(sqlite().unixSecondsSql('`t`')).toBe(
"CAST(strftime('%s', `t`) AS INTEGER)",
);
expect(mysql().unixSecondsSql('`t`')).toBe('UNIX_TIMESTAMP(`t`)');
expect(postgres().unixSecondsSql('`t`')).toBe(
'EXTRACT(EPOCH FROM `t`)::bigint',
);
});
it('reads the database clock as epoch milliseconds in each dialect', () => {
expect(sqlite().nowMsExpr()).toBe(
"CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
);
expect(mysql().nowMsExpr()).toBe(
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
);
expect(postgres().nowMsExpr()).toBe(
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
);
});
it('describes a stored timestamp the way each driver reads it back', () => {
const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');
@@ -318,13 +296,3 @@ describe('AbstractDatabaseClient — SQL generation', () => {
);
});
});
it('builds an insert for batchWrite and shares it with insert', async () => {
const client = new FixtureDatabaseClient([]);
const entry = client.insertEntry('widget', { name: "reader's", enabled: true });
expect(entry.values).toEqual(["reader's", true]);
expect(entry.statement).toContain('VALUES (?, ?)');
await client.insert('widget', { name: "reader's", enabled: true });
expect(client.writes).toEqual([{ query: entry.statement, params: entry.values }]);
});
+7 -44
View File
@@ -110,23 +110,14 @@ export class AbstractDatabaseClient extends PuterClient {
tableName: string,
data: Record<string, unknown>,
): Promise<WriteResult> {
const entry = this.insertEntry(tableName, data);
return this.write(
entry.statement + this.returningIdClause(),
entry.values,
);
}
/** An insert that can join other writes in a transaction. */
insertEntry(tableName: string, data: Record<string, unknown>): BatchEntry {
const cols = Object.keys(data);
return {
statement:
`INSERT INTO ${this.quoteIdentifier(tableName)} ` +
`(${cols.map((c) => this.quoteIdentifier(c)).join(', ')}) ` +
`VALUES (${cols.map(() => '?').join(', ')})`,
values: Object.values(data),
};
const values = Object.values(data);
const sql =
`INSERT INTO ${this.quoteIdentifier(tableName)} ` +
`(${cols.map((c) => this.quoteIdentifier(c)).join(', ')}) ` +
`VALUES (${cols.map(() => '?').join(', ')})` +
this.returningIdClause();
return this.write(sql, values);
}
/** Whether `read()` goes to a separate replica rather than the primary. */
@@ -397,34 +388,6 @@ export class AbstractDatabaseClient extends PuterClient {
});
}
/**
* A timestamp column as unix seconds, computed by the database: the mysql
* driver reads a stored UTC timestamp as local time, so parsing it in JS
* can be off by the server's UTC offset.
*/
unixSecondsSql(column: string): string {
return this.case({
postgres: `EXTRACT(EPOCH FROM ${column})::bigint`,
mysql: `UNIX_TIMESTAMP(${column})`,
otherwise: `CAST(strftime('%s', ${column}) AS INTEGER)`,
});
}
/**
* The database's own clock as epoch milliseconds, so nodes whose clocks
* disagree still agree on it. The mysql form ignores the session time zone,
* so a DST change can't shift it.
*/
nowMsExpr(): string {
return this.case({
sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
postgres:
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
otherwise:
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
});
}
/**
* What a read of a timestamp column holding `seconds` returns: sqlite gives
* back the UTC text it stored, mysql2 and pg a `Date` read in node's time
@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
/** Highest schema version the migration table can reach. */
const CURRENT_SCHEMA_VERSION = 98;
const CURRENT_SCHEMA_VERSION = 93;
/**
* These suites migrate real files on disk. Idle they finish in well under a
@@ -673,38 +673,6 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
});
});
it('adds kv_share_handles.notify from 0099, off for every existing share', async () => {
const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-notify-'));
const path = join(dir, 'puter.sqlite');
try {
const before = new SqliteDatabaseClient(
sqliteConfig({ inMemory: false, path, targetVersion: 94 }),
);
await before.onServerStart();
await before.write(
'INSERT INTO `kv_share_handles` (`handle`, `owner_user_id`, ' +
'`grantee_user_id`, `app_uid`, `key_prefix`, `permission`, ' +
'`created_at`) VALUES (?, 1, 1, ?, ?, ?, 0)',
['kvh-before-notify', 'os-global', 'a:', 'kv-share:x'],
);
before.onServerShutdown();
const after = new SqliteDatabaseClient(
sqliteConfig({ inMemory: false, path }),
);
await after.onServerStart();
await expect(
after.read(
'SELECT `notify` FROM `kv_share_handles` WHERE `handle` = ?',
['kvh-before-notify'],
),
).resolves.toEqual([{ notify: 0 }]);
after.onServerShutdown();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it('indexes notification.created_at for the retention sweep', async () => {
const rows = await client.read('PRAGMA index_list(`notification`)');
expect(rows.map((r) => String(r.name))).toContain(
@@ -712,97 +680,6 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
);
});
it('indexes notification.replaced_at per mailbox slice for catch-up', async () => {
const columns = await client.read(
'PRAGMA index_info(`idx_notification_replaced`)',
);
expect(columns.map((c) => String(c.name))).toEqual([
'user_id',
'audience',
'app_uid',
'replaced_at',
]);
});
it('keeps notification.send_key unique', async () => {
const rows = await client.read('PRAGMA index_list(`notification`)');
expect(
rows.find((r) => r.name === 'idx_notification_send_key'),
).toMatchObject({ unique: 1 });
const columns = await client.read(
'PRAGMA index_info(`idx_notification_send_key`)',
);
expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
});
it('creates event_user_handlers with its unique and holder indexes', async () => {
const columns = await client.read(
'PRAGMA table_info(`event_user_handlers`)',
);
expect(columns.map((c) => String(c.name))).toEqual([
'id',
'app_uid',
'holder_user_id',
'name',
'source',
'source_hash',
'created_at',
'updated_at',
'scope_since',
]);
const indexes = await client.read(
'PRAGMA index_list(`event_user_handlers`)',
);
expect(
indexes.find(
(i) => i.name === 'idx_event_user_handlers_app_holder_name',
),
).toMatchObject({ unique: 1 });
expect(
indexes.find((i) => i.name === 'idx_event_user_handlers_holder_app'),
).toMatchObject({ unique: 0 });
const columnsOf = async (index: string) =>
(await client.read(`PRAGMA index_info(\`${index}\`)`)).map((c) =>
String(c.name),
);
expect(await columnsOf('idx_event_user_handlers_app_holder_name')).toEqual(
['app_uid', 'holder_user_id', 'name'],
);
expect(await columnsOf('idx_event_user_handlers_holder_app')).toEqual([
'holder_user_id',
'app_uid',
]);
});
it('leaves event_handlers as it was', async () => {
const columns = await client.read('PRAGMA table_info(`event_handlers`)');
expect(columns.map((c) => String(c.name))).toEqual([
'id',
'app_uid',
'name',
'source',
'source_hash',
'created_at',
'updated_at',
]);
});
it('adds event_subscriptions.handler_scope, app scope for existing rows', async () => {
const columns = await client.read(
'PRAGMA table_info(`event_subscriptions`)',
);
expect(columns.find((c) => c.name === 'handler_scope')).toMatchObject({
notnull: 1,
dflt_value: '0',
});
const indexes = await client.read(
'PRAGMA index_list(`event_subscriptions`)',
);
expect(indexes.map((i) => String(i.name))).toContain(
'idx_event_subscriptions_holder',
);
});
it('leaves an already-migrated database untouched on a second boot', async () => {
const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
const path = join(dir, 'nested', 'puter.sqlite');
@@ -132,11 +132,6 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
[90, ['0095_kv-share-handle-levels.sql']],
[91, ['0096_kv-share-handle-quota-indexes.sql']],
[92, ['0097_user-to-user-holder-permission-index.sql']],
[93, ['0098_notification-groups.sql']],
[94, ['0099_kv-share-handle-notify.sql']],
[95, ['0100_notification-send-key.sql']],
[96, ['0101_user-scoped-handlers.sql']],
[97, ['0102_notification-scope-id-index.sql']],
];
export class SqliteDatabaseClient extends AbstractDatabaseClient {
@@ -1,53 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification`.`group_key` and `replaced_at`. See
-- sqlite/0098_notification-groups.sql for the rationale. The key is a hex
-- digest, so ascii and compared byte for byte. No per-file applied-state
-- tracking, so every step is guarded and a replay does nothing.
CALL _puter_add_col('notification', 'group_key', '`group_key` char(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
CALL _puter_add_col('notification', 'replaced_at', '`replaced_at` bigint DEFAULT NULL');
DROP PROCEDURE IF EXISTS _puter_add_notification_group_indexes;
DELIMITER //
CREATE PROCEDURE _puter_add_notification_group_indexes()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'notification'
AND INDEX_NAME = 'idx_notification_group'
) THEN
ALTER TABLE `notification` ADD INDEX `idx_notification_group`
(`user_id`, `app_uid`, `group_key`);
END IF;
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'notification'
AND INDEX_NAME = 'idx_notification_replaced'
) THEN
ALTER TABLE `notification` ADD INDEX `idx_notification_replaced`
(`user_id`, `audience`, `app_uid`, `replaced_at`);
END IF;
END//
DELIMITER ;
CALL _puter_add_notification_group_indexes();
DROP PROCEDURE IF EXISTS _puter_add_notification_group_indexes;
@@ -1,23 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Notifications through a share handle. See
-- sqlite/0099_kv-share-handle-notify.sql for the column. No per-file
-- applied-state tracking, so it goes through _puter_add_col and a replay does
-- nothing.
CALL _puter_add_col('kv_share_handles', 'notify', '`notify` tinyint(1) NOT NULL DEFAULT 0');
@@ -1,43 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
-- rationale. The key is a uuid, so ascii and compared byte for byte. No
-- per-file applied-state tracking, so every step is guarded and a replay does
-- nothing.
CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
DELIMITER //
CREATE PROCEDURE _puter_add_notification_send_key_index()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'notification'
AND INDEX_NAME = 'idx_notification_send_key'
) THEN
ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
(`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
END IF;
END//
DELIMITER ;
CALL _puter_add_notification_send_key_index();
DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
@@ -1,66 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Handlers an app's user published into their own scope, kept apart from
-- `event_handlers` so code that predates them never reads them. See
-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
--
-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline,
-- and a guarded column add. There is no per-file applied-state tracking, so a
-- replay has to be a no-op.
CREATE TABLE IF NOT EXISTS `event_user_handlers` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
-- Matches `apps`.`uid` exactly, charset included, so an equality against
-- one never falls back to a conversion.
`app_uid` char(40) CHARACTER SET latin1 COLLATE latin1_swedish_ci NOT NULL,
`holder_user_id` int unsigned NOT NULL,
`name` varchar(128) NOT NULL,
`source` mediumtext NOT NULL,
`source_hash` char(64) NOT NULL,
`created_at` bigint NOT NULL,
`updated_at` bigint NOT NULL,
-- When the scope first held a handler; kept across handler churn, so the
-- rent sweep sees the scope's age, not its oldest remaining handler's.
`scope_since` bigint NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_event_user_handlers_app_holder_name` (`app_uid`, `holder_user_id`, `name`),
KEY `idx_event_user_handlers_holder_app` (`holder_user_id`, `app_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- Guarded here rather than through `_puter_add_col`, so the file also applies
-- on its own to a database that never ran the earlier migrations.
DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;
DELIMITER //
CREATE PROCEDURE _puter_add_event_subscriptions_handler_scope()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'event_subscriptions'
AND COLUMN_NAME = 'handler_scope'
) THEN
ALTER TABLE `event_subscriptions`
ADD COLUMN `handler_scope` tinyint unsigned NOT NULL DEFAULT 0,
ALGORITHM=INSTANT;
END IF;
END//
DELIMITER ;
CALL _puter_add_event_subscriptions_handler_scope();
DROP PROCEDURE IF EXISTS _puter_add_event_subscriptions_handler_scope;
@@ -1,41 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification` (`user_id`, `audience`, `app_uid`, `id`). See
-- sqlite/0102_notification-scope-id-index.sql for the rationale. No per-file
-- applied-state tracking, so the index is added only when missing and a
-- replay does nothing.
DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;
DELIMITER //
CREATE PROCEDURE _puter_add_notification_scope_id_index()
BEGIN
IF NOT EXISTS (
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'notification'
AND INDEX_NAME = 'idx_notification_scope_id'
) THEN
ALTER TABLE `notification` ADD INDEX `idx_notification_scope_id`
(`user_id`, `audience`, `app_uid`, `id`), ALGORITHM=INPLACE, LOCK=NONE;
END IF;
END//
DELIMITER ;
CALL _puter_add_notification_scope_id_index();
DROP PROCEDURE IF EXISTS _puter_add_notification_scope_id_index;
@@ -1,29 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- notification.group_key and replaced_at. See
-- sqlite/0098_notification-groups.sql for the rationale. Idempotent via IF NOT
-- EXISTS.
ALTER TABLE notification ADD COLUMN IF NOT EXISTS group_key varchar(64) DEFAULT NULL;
ALTER TABLE notification ADD COLUMN IF NOT EXISTS replaced_at bigint DEFAULT NULL;
CREATE INDEX IF NOT EXISTS idx_notification_group
ON notification (user_id, app_uid, group_key);
CREATE INDEX IF NOT EXISTS idx_notification_replaced
ON notification (user_id, audience, app_uid, replaced_at);
@@ -1,22 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Notifications through a share handle. See
-- sqlite/0099_kv-share-handle-notify.sql for the column. Idempotent via
-- IF NOT EXISTS; there is no per-file applied-state tracking.
ALTER TABLE kv_share_handles ADD COLUMN IF NOT EXISTS notify smallint NOT NULL DEFAULT 0;
@@ -1,24 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
-- rationale. Idempotent via IF NOT EXISTS.
ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
ON notification (send_key);
@@ -1,39 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Handlers an app's user published into their own scope. See
-- sqlite/0101_user-scoped-handlers.sql for the column rationale.
-- Idempotent via IF NOT EXISTS.
CREATE TABLE IF NOT EXISTS event_user_handlers (
id BIGSERIAL PRIMARY KEY,
app_uid VARCHAR(40) NOT NULL,
holder_user_id INTEGER NOT NULL,
name VARCHAR(128) NOT NULL,
source TEXT NOT NULL,
source_hash VARCHAR(64) NOT NULL,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
scope_since BIGINT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_event_user_handlers_app_holder_name
ON event_user_handlers (app_uid, holder_user_id, name);
CREATE INDEX IF NOT EXISTS idx_event_user_handlers_holder_app
ON event_user_handlers (holder_user_id, app_uid);
ALTER TABLE event_subscriptions ADD COLUMN IF NOT EXISTS handler_scope smallint NOT NULL DEFAULT 0;
@@ -1,23 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- notification (user_id, audience, app_uid, id). See
-- sqlite/0102_notification-scope-id-index.sql for the rationale. Idempotent via
-- IF NOT EXISTS.
CREATE INDEX IF NOT EXISTS idx_notification_scope_id
ON notification (user_id, audience, app_uid, id);
@@ -1,38 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- What app message groups need from `notification`:
--
-- - `group_key` : a sha256 hex digest of an app message's `group`, so the
-- notification a newer one replaces is found by index.
-- Null on every other row, and on rows written before
-- this, which therefore never match.
-- - `replaced_at` : epoch milliseconds of the last replacement, null for a
-- row never replaced. Live and fetched copies carry it as
-- their version, so a client can tell a replacement from a
-- copy it already has. Indexed per mailbox slice, so a
-- catch-up reads the rows replaced since its cursor
-- without scanning the ones that never were.
ALTER TABLE `notification` ADD COLUMN `group_key` TEXT DEFAULT NULL;
ALTER TABLE `notification` ADD COLUMN `replaced_at` INTEGER DEFAULT NULL;
CREATE INDEX IF NOT EXISTS `idx_notification_group`
ON `notification` (`user_id`, `app_uid`, `group_key`);
CREATE INDEX IF NOT EXISTS `idx_notification_replaced`
ON `notification` (`user_id`, `audience`, `app_uid`, `replaced_at`);
@@ -1,23 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Whether the two parties of a share may send each other notifications through
-- its handle. Off by default, so every existing share reads as not allowing
-- them.
ALTER TABLE `kv_share_handles`
ADD COLUMN `notify` INTEGER NOT NULL DEFAULT 0;
@@ -1,27 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- `notification`.`send_key`: the key of the delayed send (a `deferred`
-- schedule) that wrote this row or last replaced it, derived from the
-- schedule and its slot. Unique, so one slot can never write two rows, and a
-- send that crashed after writing finds its row again by it. Null on every row
-- a delayed send never touched.
ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
ON `notification` (`send_key`);
@@ -1,48 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Handlers an app's user published into their own scope, in a table of their
-- own so `event_handlers` stays the app owner's set. Same columns as
-- `event_handlers` (see 0076_event-handlers.sql) plus the holder.
--
-- - `holder_user_id` : the user whose scope this is. A name is unique per
-- (app, holder), so the same name can live in the app's
-- set and in each holder's.
-- - `(holder_user_id, app_uid)` : a holder's scopes, for listing and for the
-- rent count.
--
-- `event_subscriptions.handler_scope` says which table a row's `handler_name`
-- refers to: 0 the app's set, 1 the row holder's scope.
CREATE TABLE IF NOT EXISTS `event_user_handlers` (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"app_uid" TEXT NOT NULL,
"holder_user_id" INTEGER NOT NULL,
"name" TEXT NOT NULL,
"source" TEXT NOT NULL,
"source_hash" TEXT NOT NULL,
"created_at" INTEGER NOT NULL, -- unix seconds
"updated_at" INTEGER NOT NULL,
"scope_since" INTEGER NOT NULL -- unix seconds; the scope's first handler, kept across churn
);
CREATE UNIQUE INDEX IF NOT EXISTS `idx_event_user_handlers_app_holder_name`
ON `event_user_handlers` (`app_uid`, `holder_user_id`, `name`);
CREATE INDEX IF NOT EXISTS `idx_event_user_handlers_holder_app`
ON `event_user_handlers` (`holder_user_id`, `app_uid`);
ALTER TABLE `event_subscriptions` ADD COLUMN `handler_scope` INTEGER NOT NULL DEFAULT 0;
@@ -1,22 +0,0 @@
-- Copyright (C) 2024-present Puter Technologies Inc.
--
-- This file is part of Puter.
--
-- Puter is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as published
-- by the Free Software Foundation, either version 3 of the License, or
-- (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General Public License
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
-- Index a mailbox slice by row id, so a fetch finds the newest row of a slice,
-- and pages forward through one, with a seek rather than a scan of the slice.
CREATE INDEX IF NOT EXISTS `idx_notification_scope_id`
ON `notification` (`user_id`, `audience`, `app_uid`, `id`);
+4 -63
View File
@@ -75,15 +75,6 @@ export type EventMap = {
serverStart: Record<string, never>;
serverPrepareShutdown: Record<string, never>;
serverShutdown: Record<string, never>;
/**
* Emitted by a host extension that knows this server will restart soon.
* Long-lived connections ask their clients to reconnect; nothing is refused
* or closed, so a false alarm is harmless.
*/
serverRestartAnnounced: {
/** How long until the server stops, when known. */
closeInMs?: number;
};
// ---- AI ----
'ai.prompt.validate': {
@@ -200,16 +191,6 @@ export type EventMap = {
};
// ---- Auth / signup ----
/**
* Which signup methods are open, emitted via `emitAndWait` before every
* signup and whenever the GUI reads the provider list. A listener sets
* `oidcOnly` to refuse form and temp signups, leaving OIDC and team
* provisioning open; `message` is what a refused form signup is told.
*/
'puter.signup.policy': {
oidcOnly: boolean;
message: string | null;
};
'puter.signup.validate': {
allow: boolean;
email?: string;
@@ -333,8 +314,6 @@ export type EventMap = {
// the abuse extension cap sends per device (across accounts) so a
// device farm can't dodge the per-account cap with fresh signups.
device_fingerprint: string | null;
// Browser-signals dispatch id the client sent, or null.
dispatch_id: string | null;
allowed: boolean;
reason: string | null;
// With `allowed` false, offer the card fallback in place of SMS. Only
@@ -556,14 +535,6 @@ export type EventMap = {
'kv.share.retired': { handles: string[]; userId: number };
// ---- Notifications ----
/** An app message was acknowledged after its state committed. */
'notif.acknowledged': {
userId: number;
uid: string;
type: string;
audience: string;
appUid: string;
};
/**
* A notification row exists. Emitted after the insert commits — nothing
* pushes a uid before there is a row to name — and carries the scope tuple
@@ -578,8 +549,6 @@ export type EventMap = {
appUid: string | null;
value: Record<string, unknown>;
createdAt: number;
/** An earlier notification replaced in place; `createdAt` is when. */
replaced?: true;
/** Events handler runs behind whatever created it. */
handlerDepth?: number;
};
@@ -597,12 +566,8 @@ export type EventMap = {
value: Record<string, unknown>;
/** An earlier notification rewritten in place. */
updated?: true;
/** Replaced an earlier notification in its group. */
replaced?: true;
/** Names this one send, for a receipt asked about it. */
sendId: string;
/** The same `sendId` may be emitted again, as a delayed send is. */
replayable?: true;
};
/**
* One notification, addressed at whatever sockets each region holds for the
@@ -612,7 +577,7 @@ export type EventMap = {
*/
'outer.notif.delivery': {
userId: number;
wire: 'notif.message' | 'notif.unreads' | 'notif.ack' | 'notif.removed';
wire: 'notif.message' | 'notif.unreads' | 'notif.ack';
response: unknown;
};
/**
@@ -686,18 +651,9 @@ export type EventMap = {
// and going away (1→0, whether by removing the last handler or by the
// destroy route). `actor.user` is the app's owner, not the caller — a
// developer session publishing for an app it owns is the common case, but
// billing follows ownership. For `scope: 'user'` the unit is one user's
// handlers in one app instead, and `actor.user` is that user.
'events.worker.create': {
actor: Actor;
appUid: string;
scope: 'app' | 'user';
};
'events.worker.destroy': {
actor: Actor;
appUid: string;
scope: 'app' | 'user';
};
// billing follows ownership.
'events.worker.create': { actor: Actor; appUid: string };
'events.worker.destroy': { actor: Actor; appUid: string };
// ---- Outer / GUI broadcast ----
'outer.cacheUpdate': {
@@ -787,7 +743,6 @@ export type EventMap = {
'outer.gui.item.renamed': GuiEvent;
'outer.gui.notif.ack': GuiEvent<{ uid: string }>;
'outer.gui.notif.message': GuiEvent<{ uid: string; notification: unknown }>;
'outer.gui.notif.removed': GuiEvent<{ uid: string }>;
'outer.gui.notif.unreads': GuiEvent<{
unreads: { uid: string; notification: unknown }[];
}>;
@@ -983,9 +938,6 @@ export type AiCostFactorEvent = {
*/
export type LifecyclePhase = 'before' | 'after' | 'error' | 'reject';
/** Status a vetoing `before` listener may answer with. */
export type LifecycleRejectStatus = 403 | 429;
/**
* Payload for `driver.<iface>.<method>.<phase>` events.
*
@@ -1015,8 +967,6 @@ export type DriverMethodLifecycleEvent = {
allow?: boolean;
/** Optional human-readable reason surfaced to the caller when vetoed. */
rejectReason?: string;
/** Status for a veto: 403 (default) or 429 for a throttle. */
rejectStatus?: LifecycleRejectStatus;
};
/**
@@ -1051,8 +1001,6 @@ export type RouteLifecycleEvent = {
allow?: boolean;
/** Optional human-readable reason surfaced to the caller when vetoed. */
rejectReason?: string;
/** Status for a veto: 403 (default) or 429 for a throttle. */
rejectStatus?: LifecycleRejectStatus;
};
export type EventKey = keyof EventMap & string;
@@ -1080,13 +1028,6 @@ export type NotifDeliveryOutcome =
/** One socket's receipt for a live notification. */
export interface NotifDeliveryResponder {
visibilityState: 'visible' | 'hidden' | 'unknown';
/** The page's window has focus. Absent from older clients. */
focused?: boolean;
/**
* The page shows its own browser notification while hidden or unfocused.
* Absent from older clients.
*/
showsOwn?: boolean;
}
export type EventMetadata = {
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -42,10 +42,6 @@ import {
adminOnlyGate,
requireUserActorGate,
} from '../../core/http/middleware/gates.js';
import {
configureRateLimit,
listConfiguredRateLimitBackends,
} from '../../core/http/middleware/rateLimit.js';
import {
ROUTES_METADATA_KEY,
type CollectedRoute,
@@ -1757,40 +1753,6 @@ describe('AuthController.handleLoginOtp + handleLoginRecoveryCode', () => {
).rejects.toMatchObject({ statusCode: 400 });
});
it('refuses the sign-in while the used-token store is down', async () => {
const { user, secret, otpToken } = await startOtpLogin();
const token = await otpToken();
const { default: backend } = listConfiguredRateLimitBackends();
configureRateLimit({
default: 'redis',
redis: {
multi: () => {
throw new Error('store down');
},
},
});
const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
const res = makeRes();
try {
await expect(
controller.handleLoginOtp(
makeReq({ token, code: liveTotp(user.username, secret) }),
res,
),
).rejects.toThrow('store down');
} finally {
logged.mockRestore();
configureRateLimit({
default: backend,
redis: server.clients.redis,
kv: server.stores.kv,
metering: server.services.metering,
});
}
expect(res.body).toBeUndefined();
});
it('a wrong code leaves the token usable for a retry', async () => {
const { user, secret, otpToken } = await startOtpLogin();
const token = await otpToken();
@@ -5132,26 +5094,6 @@ describe('AuthController phone verification — staging & reuse', () => {
expect(staged).toBe('+14155550123');
});
it('makes an email-gate-last account verify its card before its phone', async () => {
const { user, actor } = await makeUserAndActor({
requires_phone_verification: 1,
requires_card_verification: 1,
});
await server.stores.user.updateMetadata(user.id, {
emailGateLast: true,
});
const prelude = stubPrelude();
await withClients({ prelude }, async () => {
await expect(
controller.handleSendConfirmPhone(
makeReq({ phone: '+14155550123' }, { actor }),
makeRes(),
),
).rejects.toMatchObject({ statusCode: 409 });
});
expect(prelude.createVerification).not.toHaveBeenCalled();
});
it('forwards an abuse-extension veto as 429 with the opaque reason, and sends nothing', async () => {
const { user, actor } = await makeUserAndActor();
const emitAndWait = vi.fn(
@@ -5217,7 +5159,7 @@ describe('AuthController phone verification — staging & reuse', () => {
);
});
it('carries the client dispatch id on the check and sent events', async () => {
it('carries the client dispatch id on the sent signal', async () => {
const { actor } = await makeUserAndActor();
const emit = vi.fn();
const emitAndWait = vi.fn(async () => {});
@@ -5238,11 +5180,6 @@ describe('AuthController phone verification — staging & reuse', () => {
expect.objectContaining({ dispatch_id: 'disp-123' }),
expect.anything(),
);
expect(emitAndWait).toHaveBeenCalledWith(
'puter.phone-verification.check',
expect.objectContaining({ dispatch_id: 'disp-123' }),
expect.anything(),
);
});
it('does NOT emit the sent signal when the send fails upstream', async () => {
@@ -7750,46 +7687,6 @@ describe('AuthController permission revokes', () => {
);
expect(res.body).toEqual({});
});
it('revoke-user-user: refuses a key-value share grant and leaves it standing', async () => {
const { actor: ownerActor, user: owner } = await makeUserAndActor();
const { user: holder } = await makeUserAndActor();
const permissions = ['kv-share', 'kv-share-read', 'kv-share-write'].map(
(root) => `${root}:${owner.uuid}:os-global:notes:${uuidv4()}`,
);
for (const permission of permissions)
await server.stores.permission.upsertUserUserPerm(
holder.id,
owner.id,
permission,
{},
);
for (const permission of permissions)
await expect(
inCtx(ownerActor, () =>
controller.handleRevokeUserUser(
makeReq(
{ target_username: holder.username, permission },
{ actor: ownerActor },
),
makeRes(),
),
),
).rejects.toMatchObject({
statusCode: 400,
message: expect.stringContaining('puter.kv.unshare()'),
});
expect(
(
await server.stores.permission.findUserUserPermsByIssuer(
holder.id,
owner.id,
permissions,
)
).sort(),
).toEqual([...permissions].sort());
});
});
// ── Permission checks + listing ────────────────────────────────────
+8 -59
View File
@@ -66,7 +66,6 @@ import {
isCardFallbackEnabled,
phoneAttemptsKey,
} from '../../util/cardFallback.js';
import { isEmailGateLast } from '../../util/verificationOrder.js';
import { sessionCookieFlags } from '../../util/cookieFlags.js';
import { cleanEmail, isStorableEmail } from '../../util/email.js';
import { isGodmodeApp } from '../../util/godmodeApps.js';
@@ -1221,18 +1220,6 @@ export class AuthController extends PuterController {
}
}
/** The email-gate-last confirmation email; a failure never fails the gate. */
private async sendDeferredConfirmation(userId: number): Promise<void> {
try {
await this.services.signup.sendDeferredConfirmationEmail(userId);
} catch (e) {
console.warn(
'[verification] deferred confirmation email failed:',
e,
);
}
}
private async isCardFallbackEligible(user: {
id: number;
requires_phone_verification?: boolean | number | null;
@@ -1278,12 +1265,6 @@ export class AuthController extends PuterController {
throw new HttpError(403, 'Account suspended.', {
legacyCode: 'account_suspended',
});
if (isEmailGateLast(user) && user.requires_card_verification)
throw new HttpError(
409,
'Card verification must be completed first.',
{ legacyCode: 'conflict' },
);
if (!this.clients.prelude?.isConfigured())
throw await this.smsSendError(
503,
@@ -1354,7 +1335,6 @@ export class AuthController extends PuterController {
user_uid: user.uuid,
phone: parsed.e164,
device_fingerprint: req.deviceFingerprint ?? null,
dispatch_id: dispatchId ?? null,
allowed: true,
reason: null as string | null,
card_fallback_instead: false,
@@ -1593,7 +1573,6 @@ export class AuthController extends PuterController {
requires_phone_verification: 0,
phone: pendingPhone,
});
await this.sendDeferredConfirmation(user.id);
// Run the verified-event listeners synchronously (emitAndWait, not
// fire-and-forget emit) so a carrier-based card-verification waiver in
@@ -1674,11 +1653,7 @@ export class AuthController extends PuterController {
res.json({ card_verified: true });
return;
}
if (
user.requires_phone_verification &&
!fallbackEligible &&
!isEmailGateLast(user)
)
if (user.requires_phone_verification && !fallbackEligible)
throw new HttpError(
409,
'Phone verification must be completed first.',
@@ -1741,7 +1716,6 @@ export class AuthController extends PuterController {
await this.stores.user.update(user.id, {
requires_card_verification: 0,
});
await this.sendDeferredConfirmation(user.id);
res.json({ card_verified: true, disabled: true });
return;
}
@@ -1801,11 +1775,7 @@ export class AuthController extends PuterController {
res.json({ card_verified: true });
return;
}
if (
user.requires_phone_verification &&
!fallbackEligible &&
!isEmailGateLast(user)
)
if (user.requires_phone_verification && !fallbackEligible)
throw new HttpError(
409,
'Phone verification must be completed first.',
@@ -1846,7 +1816,6 @@ export class AuthController extends PuterController {
await this.stores.user.update(user.id, {
requires_card_verification: 0,
});
await this.sendDeferredConfirmation(user.id);
res.json({ card_verified: true, disabled: true });
return;
}
@@ -1871,7 +1840,6 @@ export class AuthController extends PuterController {
requires_card_verification: 0,
...(clearedPhoneGate ? { requires_phone_verification: 0 } : {}),
});
await this.sendDeferredConfirmation(user.id);
// Awaited like `user.phone-verified`, so whatever a listener grants on
// verification is in place before the client refreshes. emitAndWait
@@ -3336,14 +3304,6 @@ export class AuthController extends PuterController {
{ legacyCode: 'bad_request' },
);
}
// Only unsharing retires the handle along with its grant; withdrawing
// the grant alone would leave the share listed and dead.
if (typeof permission === 'string' && isKvSharePermission(permission))
throw new HttpError(
400,
'Stop a key-value share with `puter.kv.unshare()` (`DELETE /kv/shares/:handle`)',
{ legacyCode: 'bad_request' },
);
await this.services.permission.revokeUserUserPermission(
req.actor!,
target_username,
@@ -3850,9 +3810,8 @@ export class AuthController extends PuterController {
let app = await this.stores.app.getByUid(app_uid);
if (!app && resolvedFromOrigin) {
// Hosted-subdomain and custom-domain origins get the site owner
// stamped as the app's creator at bootstrap; external origins
// stay unowned.
// Hosted-subdomain origins get the site owner stamped as the
// app's creator at bootstrap; external origins stay unowned.
// Canonical origin only, so alternate hosts share one row.
const canonicalOrigin =
this.services.auth.canonicalizeOrigin(origin);
@@ -3868,10 +3827,9 @@ export class AuthController extends PuterController {
// An origin's uid is a deterministic uuidv5, so a deleted app
// reappears here under the identical uid. Withdraw any cross-app
// data grants left pointing at it before this new row can inherit
// consent the user gave its predecessor. Only `createFromOrigin`
// reuses a uid, here or wherever else an origin app is bootstrapped:
// `AppStore.create` mints a random uuid4, which no deleted app can
// ever hold again.
// consent the user gave its predecessor. Only *this* path can reuse
// a uid: `AppStore.create` mints a random uuid4, which no deleted
// app can ever hold again.
//
// Called directly rather than through `app.changed`: the token is
// issued below, so this has to be able to stop that, and
@@ -3882,9 +3840,6 @@ export class AuthController extends PuterController {
app_uid,
'uid reused by a new app',
);
// Key-value shares the predecessor left read as gone already,
// being older than this row; this only cleans them up.
this.services.kvShare.retireSharesOlderThanApp(app);
}
if (!app) {
throw new HttpError(404, `App ${app_uid} does not exist`, {
@@ -4681,19 +4636,13 @@ export class AuthController extends PuterController {
await checkRateLimit(`${scope}:${userUid}`, limit, window);
}
/**
* An `otp-login` JWT completes one sign-in; a second redemption is refused.
* Fails closed: while the store can't say whether the token was used, the
* sign-in is refused rather than letting a captured token replay.
*/
/** An `otp-login` JWT completes one sign-in; a second redemption is refused. */
async #consumeOtpLoginToken(token: string): Promise<void> {
const digest = crypto.createHash('sha256').update(token).digest('hex');
const firstUse = await checkRateLimit(
`login-otp-token-used:${digest}`,
1,
OTP_LOGIN_TOKEN_USED_WINDOW_MS,
undefined,
{ failOpen: false },
);
if (!firstUse) {
throw new HttpError(400, 'Invalid token.', {
@@ -19,7 +19,6 @@
import type { Request, Response } from 'express';
import { Controller, Post } from '../../core/http/decorators.js';
import { schedulesUnavailable } from '../../core/schedules.js';
import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
import type { ForwardBatch } from '../../services/events/forwardQueue.js';
import { PuterController } from '../types.js';
@@ -109,42 +108,6 @@ export class BroadcastController extends PuterController {
);
res.status(200).json({ ok: true, ...reply });
}
/**
* Schedule changes a region made for schedules this one owns, handed to
* `services.schedules`; `events_disabled` on a server without it.
*/
@Post('/events/schedules', { subdomain: '*' })
async schedules(req: Request, res: Response): Promise<void> {
const broadcast = this.services.broadcast as unknown as
BroadcastService | undefined;
if (!broadcast) {
res.status(503).json({
error: { message: 'Broadcast service not registered' },
});
return;
}
const verified = await broadcast.verifySignedRequest(
req.rawBody,
broadcastHeaders(req),
);
if (!verified.ok) {
res.status(verified.status ?? 400).json({
error: { message: verified.message ?? 'Bad request' },
});
return;
}
if (verified.info?.ignored) {
res.status(200).json({ ok: true, ...verified.info });
return;
}
const schedules = this.services.schedules;
if (!schedules) throw schedulesUnavailable();
const reply = await schedules.receivePeerChanges(req.body);
res.status(200).json({ ok: true, ...reply });
}
}
const broadcastHeaders = (req: Request) => {
@@ -736,42 +736,6 @@ describe('DriverController driver-method lifecycle events', () => {
rejectReason: 'blocked in test',
});
});
it('answers 429 when a before listener vetoes with rejectStatus 429', async () => {
const actor = await makeUserActor();
await grantKvPerm(actor);
const key = `throttle-${uuidv4()}`;
server.clients.event.on(
'driver.puter-kvstore.set.before',
(_k, data) => {
if ((data.args as { key?: string })?.key === key) {
data.allow = false;
data.rejectStatus = 429;
}
},
);
const req = makeReq(
{
interface: 'puter-kvstore',
method: 'set',
args: { key, value: 'v' },
},
actor,
);
await expect(
runWithContext({ actor }, () =>
routes['POST /call'](
req,
makeRes() as unknown as Response,
() => {},
),
),
).rejects.toMatchObject({
statusCode: 429,
legacyCode: 'too_many_requests',
});
});
});
describe('DriverController.#handleListInterfaces', () => {
@@ -31,7 +31,6 @@ import {
checkDriverRateLimit,
} from '../../core/http/middleware/rateLimit.js';
import type { PuterRouter } from '../../core/http/PuterRouter.js';
import type { LifecycleRejectStatus } from '../../clients/event/types.js';
import type { DriverMeta } from '../../drivers/meta.js';
import {
isCreditExhaustion,
@@ -335,8 +334,7 @@ export class DriverController extends PuterController {
if (req.actor) {
const permService = this.services.permission as unknown as
| PermissionService
| undefined;
PermissionService | undefined;
if (permService) {
// Build via PermissionUtil.join so any `:` in a driver or
// interface name is escaped — raw interpolation would let a
@@ -477,7 +475,6 @@ export class DriverController extends PuterController {
args,
allow: true as boolean,
rejectReason: undefined as string | undefined,
rejectStatus: undefined as LifecycleRejectStatus | undefined,
};
await this.clients.event?.emitAndWait(
`driver.${ifaceName}.${method}.before`,
@@ -499,12 +496,11 @@ export class DriverController extends PuterController {
},
{},
);
const throttled = beforeEvent.rejectStatus === 429;
throw new HttpError(
throttled ? 429 : 403,
403,
beforeEvent.rejectReason ??
`Blocked by policy: ${ifaceName}:${method}`,
{ legacyCode: throttled ? 'too_many_requests' : 'forbidden' },
{ legacyCode: 'forbidden' },
);
}
@@ -25,11 +25,6 @@ import { bodyRecord } from '../../core/http/requestBody.js';
import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
import {
SCHEDULE_LIST_LIMIT_CAP,
schedulesUnavailable,
type ScheduleServiceContract,
} from '../../core/schedules.js';
import { normalizeLimit } from '../../util/pagination.js';
import { PuterController } from '../types.js';
import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
@@ -64,8 +59,6 @@ const EVENTS_SCRIPT_REGEX = new RegExp(`^${EVENTS_WORKER_PREFIX}[a-f0-9]{32}$`);
/** Matches the edge dispatcher's own validation of the app uid it forwards. */
const APP_UID_REGEX = /^app-[A-Za-z0-9_-]{1,64}$/;
/** A user uuid. */
const HOLDER_REGEX = /^[A-Za-z0-9-]{1,64}$/;
@Controller('/events')
export class EventsController extends PuterController {
@@ -111,31 +104,12 @@ export class EventsController extends PuterController {
});
}
/**
* POST /events/subscriptions/update — replace one subscription's `context`
* in place. An id the caller does not hold reads as absent.
*/
@Post('/subscriptions/update', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
})
async update(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
const { sub } = await this.services.events.updateDurable(
actor,
bodyRecord(req),
);
res.json(sub);
}
/**
* GET /events/fetch — what the caller missed, one page at a time.
*
* Stateless: no subscription, no stored position, nothing written. The
* cursor comes back to the caller and goes out again as `cursor` (or
* `after`, the name it first shipped under). `from=end` with no cursor
* starts at the newest row instead of the oldest.
* `after`, the name it first shipped under).
*/
@Get('/fetch', {
subdomain: 'api',
@@ -146,16 +120,9 @@ export class EventsController extends PuterController {
async fetch(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
const query = (req.query ?? {}) as Record<string, unknown>;
const from = query.from as 'start' | 'end' | undefined;
if (from !== undefined && from !== 'start' && from !== 'end') {
throw new HttpError(400, '`from` must be `start` or `end`', {
legacyCode: 'bad_request',
});
}
const page = await this.services.events.fetchMissed(actor, {
subject: typeof query.subject === 'string' ? query.subject : '',
from,
cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
after: typeof query.after === 'string' ? query.after : undefined,
limit: normalizeLimit(query.limit, {
@@ -184,99 +151,6 @@ export class EventsController extends PuterController {
res.json({});
}
// -- Schedules ---------------------------------------------------
//
// An app's handler run on a timer, served by `services.schedules` when a
// server has it and answered `events_disabled` when it doesn't. Creating
// one needs an app context; listing, reading and removing are scoped the
// way the durable verbs are. The create and remove verbs spend the
// subscribe and unsubscribe budgets in the service, shared with those
// verbs.
#schedules(): ScheduleServiceContract {
const schedules = this.services.schedules;
if (!schedules) throw schedulesUnavailable();
return schedules;
}
/** POST /events/schedules — create a schedule for the calling app. */
@Post('/schedules', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
})
async createSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
}
/** GET /events/schedules — the caller's schedules, one page at a time. */
@Get('/schedules', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
rateLimit: EVENTS_LIST_LIMIT,
})
async listSchedules(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
const query = (req.query ?? {}) as Record<string, unknown>;
const page = await this.#schedules().list(actor, {
limit: normalizeLimit(query.limit, {
cap: SCHEDULE_LIST_LIMIT_CAP,
}),
cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
includeTotal: query.includeTotal === 'true',
});
res.json({
items: page.items,
...(page.cursor ? { cursor: page.cursor } : {}),
...(page.total !== undefined ? { total: page.total } : {}),
});
}
@Post('/schedules/update', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
})
async updateSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
const { scheduleId, changes } = bodyRecord(req);
res.json(await this.#schedules().update(actor, scheduleId, changes));
}
/**
* POST /events/schedules/remove — end one schedule. An id the caller does
* not hold reads as absent.
*/
@Post('/schedules/remove', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
})
async removeSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
res.json({});
}
/**
* GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
* caller does not hold reads as absent.
*/
@Get('/schedules/:scheduleId', {
subdomain: 'api',
requireAuth: true,
allowAccessToken: true,
rateLimit: EVENTS_LIST_LIMIT,
})
async getSchedule(req: Request, res: Response): Promise<void> {
const actor = this.#requireActor(req);
res.json(await this.#schedules().get(actor, req.params.scheduleId));
}
// -- Cross-user key-value handles --------------------------------
/**
@@ -423,7 +297,6 @@ export class EventsController extends PuterController {
handlers: await this.services.events.listHandlers(actor, {
appUid:
typeof query.appUid === 'string' ? query.appUid : undefined,
scope: query.scope,
}),
});
}
@@ -522,15 +395,6 @@ export class EventsController extends PuterController {
throw new HttpError(400, 'Missing or invalid `script`/`appUid`', {
legacyCode: 'bad_request',
});
// Absent or empty is the app's own scope.
const holder = body.holder === undefined ? '' : body.holder;
if (
typeof holder !== 'string' ||
!(holder === '' || HOLDER_REGEX.test(holder))
)
throw new HttpError(400, 'Invalid `holder`', {
legacyCode: 'bad_request',
});
const deployer = this.#eventsWorkerDeployer();
if (!deployer.enabled) {
@@ -538,25 +402,15 @@ export class EventsController extends PuterController {
return;
}
const scope = await deployer.scopeFor(appUid, holder);
if (!scope) {
res.status(404).json({ deployed: false, reason: 'no-holder' });
return;
}
const outcome = await deployer.ensure(scope, script);
const outcome = await deployer.ensure(appUid, script);
if (outcome === 'deployed') {
res.json({ deployed: true });
return;
}
// `failed`, `throttled` and `busy` are worth another attempt: the rest
// say this script is not what the app's handlers currently are.
// `failed` and `throttled` are worth another attempt: the rest say
// this script is not what the app's handlers currently are.
res.status(
outcome === 'failed' ||
outcome === 'throttled' ||
outcome === 'busy'
? 502
: 404,
outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
).json({
deployed: false,
reason: outcome,
@@ -577,22 +431,14 @@ export class EventsController extends PuterController {
override onServerStart(): void {
const deployer = this.#eventsWorkerDeployer();
if (!deployer.enabled) return;
this.services.events.useEagerDeploy((scope, script) =>
deployer.deployEagerly(scope, script),
);
if (this.config.workers?.localServer) {
this.services.events.useWorkerTransport(
new LocalEventsInvokeTransport(this.services, deployer),
);
return;
}
this.services.events.useWorkerMissHandler(
async (appUid, script, options) => {
const scope = await deployer.scopeFor(appUid, options.holder);
return scope
? deployer.ensure(scope, script, options)
: 'no-holder';
},
this.services.events.useWorkerMissHandler((appUid, script) =>
deployer.ensure(appUid, script),
);
}
+7 -69
View File
@@ -34,46 +34,23 @@ import {
EVENTS_DURABLE_SUBSCRIPTIONS_MAX,
EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP,
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
EVENTS_HANDLER_PUBLISH_LIMIT,
EVENTS_HANDLER_SOURCE_MAX_BYTES,
EVENTS_HANDLERS_PER_APP,
EVENTS_KV_HANDLES_PER_APP,
EVENTS_KV_HANDLES_PER_USER,
EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT,
EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT,
EVENTS_SINGLE_DELIVERY_LIMIT,
EVENTS_USER_HANDLERS_PER_SCOPE,
EVENTS_USER_SCOPE_SOURCE_MAX_BYTES,
EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_INVOCATION_LIMIT,
EVENTS_WORKER_SOURCE_MAX_BYTES,
RENTED_UNITS_PER_USER,
limitFor,
tiered,
type TieredLimit,
} from './limits.js';
const tiers: Array<[string, TieredLimit]> = [
[
'durable subscriptions per account',
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
],
['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
];
describe('tiered', () => {
it('gives paid plans the base, and the free and temporary tiers their own', () => {
const limit = tiered(30, 20, 10);
expect(limitFor(limit, null)).toBe(30);
expect(limitFor(limit, 'some_plan')).toBe(30);
expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
});
});
describe('the tiered subscription quotas', () => {
it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
for (const n of Object.values(tier.bySubscription))
@@ -128,9 +105,9 @@ describe('the tiered subscription quotas', () => {
});
it('holds an unrecognised plan to the paid base', () => {
expect(
limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
);
expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
);
@@ -154,9 +131,9 @@ describe('the delivery budgets', () => {
expect(
limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
).toBe(matched);
expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
evaluated,
);
expect(
limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
).toBe(evaluated);
expect(evaluated).toBeGreaterThanOrEqual(matched);
}
});
@@ -192,42 +169,3 @@ describe('the delivery budgets', () => {
}
});
});
describe('the handler limits', () => {
it('pins the app-scope numbers the quotas page lists', () => {
expect(EVENTS_HANDLERS_PER_APP).toBe(100);
expect(EVENTS_HANDLER_SOURCE_MAX_BYTES).toBe(64 * 1024);
expect(EVENTS_WORKER_SOURCE_MAX_BYTES).toBe(5 * 1024 * 1024);
expect(EVENTS_WORKER_DEPLOYS_PER_HOUR).toBe(30);
expect(EVENTS_HANDLER_PUBLISH_LIMIT.limit).toBe(60);
expect(EVENTS_HANDLER_PUBLISH_LIMIT.window).toBe(60_000);
});
it('pins the user-scope numbers the quotas page lists', () => {
expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBe(20);
expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBe(1024 * 1024);
expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBe(10);
});
it('keeps a user scope tighter than an app scope', () => {
expect(EVENTS_USER_HANDLERS_PER_SCOPE).toBeLessThan(
EVENTS_HANDLERS_PER_APP,
);
expect(EVENTS_USER_SCOPE_SOURCE_MAX_BYTES).toBeLessThan(
EVENTS_WORKER_SOURCE_MAX_BYTES,
);
expect(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR).toBeLessThan(
EVENTS_WORKER_DEPLOYS_PER_HOUR,
);
});
it('pins the rented-unit cap by plan', () => {
expect(RENTED_UNITS_PER_USER.limit).toBe(100);
expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_FREE_SUBSCRIPTION)).toBe(
16,
);
expect(limitFor(RENTED_UNITS_PER_USER, DEFAULT_TEMP_SUBSCRIPTION)).toBe(
16,
);
});
});
+19 -80
View File
@@ -53,12 +53,7 @@ export interface TieredLimit {
bySubscription: Record<string, number>;
}
/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
export const tiered = (
paid: number,
free: number,
temp: number,
): TieredLimit => ({
const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
limit: paid,
bySubscription: {
[DEFAULT_FREE_SUBSCRIPTION]: free,
@@ -98,18 +93,6 @@ export interface TieredQuota {
*/
export const EVENTS_SESSION_SUBSCRIPTIONS_PER_SOCKET = 50;
/**
* Live subscriptions one user may hold through a share handle someone else
* minted for them, across all of their connections.
*
* Those rows sit in the owner's keyspace, where each one costs a filter
* evaluation on the owner's writes. The count also takes in connections that
* dropped and haven't been reaped yet, so the cap sits far above what
* reconnects leave behind. The owner's own rows through the handle don't
* count.
*/
export const EVENTS_SESSION_SUBSCRIPTIONS_PER_KV_HANDLE = 1000;
/**
* Durable subscriptions one account may hold, across every app.
*
@@ -142,13 +125,6 @@ export const EVENTS_DURABLE_SUBSCRIPTIONS_MAX =
*/
export const SUSPENDED_ROW_TTL_DAYS = 30;
/**
* Subscriptions stopped by a revoked grant that one account keeps for `list`.
* They hold no quota slot, so past this the oldest are deleted when the account
* subscribes again, keeping its rows bounded.
*/
export const EVENTS_REVOKED_SUBSCRIPTIONS_KEPT = 500;
/**
* Subscribe calls per minute, per user.
*
@@ -228,35 +204,17 @@ export const EVENTS_KV_HANDLES_PER_APP = tiered(10_000, 2_000, 0);
export const EVENTS_KV_HANDLES_PER_HOLDER = 256;
/**
* Recipients one account may share with, per minute and per hour. A call spends
* one per recipient that doesn't already hold the share, on top of the per-call
* mint budget.
* Recipients one account may share with, per minute and per hour. A batch
* spends one per recipient, on top of the per-call mint budget.
*/
export const KV_SHARE_GRANT_LIMITS = [
userWindow('kvShare:grants:minute', 300),
userWindow('kvShare:grants:hour', 2_000, 60 * 60_000),
] as const;
/**
* Recipients one account may name in share calls per minute, existing shares
* included. Each one is looked up before the call knows whether it changes
* anything, so a client that shares again on every load is bounded here.
*/
export const KV_SHARE_LOOKUP_LIMIT = userWindow('kvShare:lookups', 1_200);
/** Recipients one `POST /kv/shares` call may name. */
export const KV_SHARE_BATCH_MAX = 50;
/**
* Notifications one party of a share may send the other through share handles,
* per minute and per day, in one app. Keeps one share from spending the
* recipient's whole notification allowance.
*/
export const KV_SHARE_NOTIFY_LIMITS = [
userWindow('kvShare:notify:minute', 30),
userWindow('kvShare:notify:day', 500, 24 * 60 * 60_000),
] as const;
// -- Handler surface -------------------------------------------------
/**
@@ -269,12 +227,6 @@ export const KV_SHARE_NOTIFY_LIMITS = [
*/
export const EVENTS_HANDLERS_PER_APP = 100;
/**
* Handlers one user may hold in their own scope of one app. Each is a script of
* its own, so the cap is tighter than the app's.
*/
export const EVENTS_USER_HANDLERS_PER_SCOPE = 20;
/** Longest a serialized handler may be. */
export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
@@ -286,9 +238,6 @@ export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024;
*/
export const EVENTS_WORKER_SOURCE_MAX_BYTES = 5 * 1024 * 1024;
/** Source one user scope may hold, all its handlers combined. */
export const EVENTS_USER_SCOPE_SOURCE_MAX_BYTES = 1024 * 1024;
/**
* Deploys one app's events worker may trigger per hour.
*
@@ -298,16 +247,6 @@ export const EVENTS_USER_SCOPE_SOURCE_MAX_BYTES = 1024 * 1024;
*/
export const EVENTS_WORKER_DEPLOYS_PER_HOUR = 30;
/** Deploys one user scope may trigger per hour. */
export const EVENTS_USER_WORKER_DEPLOYS_PER_HOUR = 10;
/**
* Units one user may rent at once: ordinary workers, user scopes, and the
* events workers of apps they own. Only creating a unit is refused; holdings
* over the cap are kept.
*/
export const RENTED_UNITS_PER_USER = tiered(100, 16, 16);
/**
* Handlers one `publishAll` may carry. A build step publishes its whole set in
* one call, and the set is capped by what an app may hold anyway.
@@ -372,11 +311,8 @@ export const EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT = tiered(512, 128, 128);
*/
export const EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT = tiered(2048, 512, 512);
/**
* Deliveries of one key-value change that carry its value inline, in the order
* the fan-out serves them. Later ones carry the key alone.
*/
export const EVENTS_KV_VALUE_DELIVERIES_PER_EVENT = 128;
/** Omit all inline KV values when the matched audience exceeds this count. */
export const EVENTS_KV_VALUE_OMIT_MATCHED_SUBSCRIPTIONS = 128;
/**
* Largest inline KV value in serialized bytes. Larger values are omitted;
@@ -468,20 +404,16 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
// a handler that answers "no" is not. Retrying on a fixed cadence turns one
// broken deploy into a permanent load on whatever is failing, so the wait
// doubles per attempt up to a ceiling, and a run of failures stops the
// subscription rather than retrying it forever. A run Puter could not start or
// finish backs off the same way but is not the handler's failure, so it never
// counts toward that run.
// subscription rather than retrying it forever.
// Wait before the first retry of a delivery a handler could not take, the
// longest wait, and the doubling between them. Shared with system jobs.
export {
deliveryBackoffMs,
EVENTS_RETRY_BASE_MS,
EVENTS_RETRY_MAX_MS,
} from '../../util/retryBackoff.js';
/** Wait before the first retry of a delivery a handler could not take. */
export const EVENTS_RETRY_BASE_MS = 2_000;
/** Longest wait between retries, however many have failed. */
export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
/**
* Handler failures in a row before a subscription is suspended. Counted per
* Failures in a row before a subscription is suspended. Counted per
* subscription and reset by the first delivery a handler takes, so an
* occasional failure never accumulates into one.
*/
@@ -494,6 +426,13 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
*/
export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
/** How long to hold a delivery whose handler has failed `attempts` times. */
export const deliveryBackoffMs = (attempts: number): number =>
Math.min(
EVENTS_RETRY_MAX_MS,
EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
);
// -- Suspended backlog -----------------------------------------------
//
// A suspended subscription stops metering but keeps what it is owed, and that
@@ -1,289 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The schedule routes are thin delegates: without `services.schedules` every
* one answers 503 `events_disabled`; with it, each hands its request over and
* answers what it returns.
*/
import type { Request, Response } from 'express';
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
it,
vi,
} from 'vitest';
import { makeActor, type Actor } from '../../core/actor.js';
import { isHttpError } from '../../core/http/HttpError.js';
import type {
ScheduleServiceContract,
ScheduleView,
} from '../../core/schedules.js';
import type { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import type { BroadcastController } from '../broadcast/BroadcastController.js';
import type { EventsController } from './EventsController.js';
let server: PuterServer;
let events: EventsController;
let broadcast: BroadcastController;
let actor: Actor;
let saved: unknown;
const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
const view = {
scheduleId: SCHEDULE_ID,
kind: 'recurring',
appUid: 'app-x',
} as ScheduleView;
const slot = () => server.services as unknown as { schedules?: unknown };
/** Put `schedules` in the slot, or leave the slot empty. */
const setSchedules = (schedules: unknown): void => {
if (schedules === undefined) delete slot().schedules;
else slot().schedules = schedules;
};
const fakeSchedules = () =>
({
onSubscriptionsChanged: vi.fn(async () => {}),
createFor: vi.fn(async () => view),
list: vi.fn(async () => ({
items: [view],
cursor: 'next',
total: 3,
})),
get: vi.fn(async () => view),
update: vi.fn(async () => view),
remove: vi.fn(async () => {}),
receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
prepareSendAt: vi.fn(),
createDeferredAppMessage: vi.fn(),
}) satisfies ScheduleServiceContract;
const req = (init: {
body?: unknown;
query?: Record<string, unknown>;
params?: Record<string, string>;
}): Request =>
({
actor,
body: init.body ?? {},
query: init.query ?? {},
params: init.params ?? {},
headers: {},
rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
}) as unknown as Request;
const res = () => {
const sent: { status: number; body: unknown } = {
status: 200,
body: undefined,
};
const response = {
json: vi.fn((body: unknown) => {
sent.body = body;
return response;
}),
status: vi.fn((code: number) => {
sent.status = code;
return response;
}),
setHeader: vi.fn(() => response),
};
return { response: response as unknown as Response, sent };
};
/** Each route, called as the router would. */
const routes: Array<[string, (response: Response) => Promise<void>]> = [
[
'POST /events/schedules',
(r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
],
[
'GET /events/schedules',
(r) => events.listSchedules(req({ query: { limit: '5' } }), r),
],
[
'GET /events/schedules/:scheduleId',
(r) =>
events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
],
[
'POST /events/schedules/update',
(r) => events.updateSchedule(req({ body: { scheduleId: SCHEDULE_ID, changes: { every: '2h' } } }), r),
],
[
'POST /events/schedules/remove',
(r) =>
events.removeSchedule(
req({ body: { scheduleId: SCHEDULE_ID } }),
r,
),
],
[
'POST /broadcast/events/schedules',
(r) =>
broadcast.schedules(
req({ body: { namespace: 'n', changes: [] } }),
r,
),
],
];
beforeAll(async () => {
server = await setupTestServer();
events = server.controllers.events as unknown as EventsController;
broadcast = server.controllers.broadcast as unknown as BroadcastController;
actor = makeActor({
user: { id: 1, uuid: 'u-1', username: 'someone' },
} as never);
saved = slot().schedules;
vi.spyOn(
server.services.broadcast,
'verifySignedRequest',
).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
});
afterEach(() => {
setSchedules(saved);
});
afterAll(async () => {
vi.restoreAllMocks();
await server?.shutdown();
});
describe('without services.schedules', () => {
it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
setSchedules(undefined);
const { response } = res();
let thrown: unknown;
try {
await call(response);
} catch (err) {
thrown = err;
}
expect(isHttpError(thrown)).toBe(true);
expect(thrown).toMatchObject({
statusCode: 503,
legacyCode: 'events_disabled',
// The designed answer, so it never pages.
noAlarm: true,
});
});
it('refuses an unsigned peer batch before saying schedules are off', async () => {
setSchedules(undefined);
vi.spyOn(
server.services.broadcast,
'verifySignedRequest',
).mockResolvedValueOnce({
ok: false,
status: 401,
message: 'bad signature',
} as never);
const { response, sent } = res();
await broadcast.schedules(
req({ body: { namespace: 'n', changes: [] } }),
response,
);
expect(sent.status).toBe(401);
});
});
describe('with services.schedules', () => {
it('creates through createFor and answers its view', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.createSchedule(req({ body: { every: 60_000 } }), response);
expect(schedules.createFor).toHaveBeenCalledWith(actor, {
every: 60_000,
});
expect(sent.body).toEqual(view);
});
it('lists with the limit capped, the cursor and includeTotal', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.listSchedules(
req({
query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
}),
response,
);
expect(schedules.list).toHaveBeenCalledWith(actor, {
limit: 200,
cursor: 'c',
includeTotal: true,
});
expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
});
it('reads one through get', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.getSchedule(
req({ params: { scheduleId: SCHEDULE_ID } }),
response,
);
expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
expect(sent.body).toEqual(view);
});
it('updates through update and answers its view', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
const changes = { every: '2h' };
await events.updateSchedule(req({ body: { scheduleId: SCHEDULE_ID, changes } }), response);
expect(schedules.update).toHaveBeenCalledWith(actor, SCHEDULE_ID, changes);
expect(sent.body).toEqual(view);
});
it('removes through remove and answers {}', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
await events.removeSchedule(
req({ body: { scheduleId: SCHEDULE_ID } }),
response,
);
expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
expect(sent.body).toEqual({});
});
it('hands a signed peer batch to receivePeerChanges', async () => {
const schedules = fakeSchedules();
setSchedules(schedules);
const { response, sent } = res();
const body = { namespace: 'n', changes: [] };
await broadcast.schedules(req({ body }), response);
expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
});
});
@@ -1,77 +0,0 @@
import type { Request, Response } from 'express';
import { describe, expect, it } from 'vitest';
import { makeActor } from '../../core/actor.js';
import type {
ScheduleServiceContract,
ScheduleView,
} from '../../core/schedules.js';
import { EventsController } from './EventsController.js';
const actor = makeActor({
user: { id: 1, uuid: 'user-1', username: 'user1' } as never,
});
const request = (body: unknown) => ({ actor, body }) as Request;
const controller = (schedules?: Pick<ScheduleServiceContract, 'update'>) =>
new EventsController(
{} as never,
{} as never,
{} as never,
{ schedules } as never,
{} as never,
);
describe('update schedule route contract', () => {
it('answers events_disabled when schedules are unavailable', async () => {
await expect(
controller().updateSchedule(
request({ scheduleId: 'id', changes: {} }),
{} as Response,
),
).rejects.toMatchObject({
legacyCode: 'events_disabled',
statusCode: 503,
noAlarm: true,
});
});
it('passes the actor, id and changes through and returns the updated view', async () => {
const view = {
scheduleId: 'id',
kind: 'recurring',
state: 'suspended',
} as ScheduleView;
const changes = { cron: '0 9 * * *', tz: 'UTC' };
const calls: unknown[][] = [];
const events = controller({
update: async (...args) => {
calls.push(args);
return view;
},
});
let sent: unknown;
await events.updateSchedule(request({ scheduleId: 'id', changes }), {
json: (value: unknown) => {
sent = value;
},
} as Response);
expect(calls).toEqual([[actor, 'id', changes]]);
expect(sent).toBe(view);
});
it('preserves validation and sending errors from the implementation', async () => {
const error = {
code: 'events_schedule_sending',
message: 'Already sending',
};
await expect(
controller({
update: async () => {
throw error;
},
}).updateSchedule(
request({ scheduleId: 'id', changes: {} }),
{} as Response,
),
).rejects.toBe(error);
});
});
@@ -25,35 +25,9 @@
import MockRedis from 'ioredis-mock';
import { describe, expect, it } from 'vitest';
import type { fetch as undiciFetch } from 'undici';
import {
DispatcherInvokeTransport,
EVENTS_DEPLOYED_HEADER,
EVENTS_DISPATCH_ERROR_HEADER,
EVENTS_HANDLED_HEADER,
EventsWorkerInvokerClient,
type EventsInvokeCall,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import { INTERNAL_DEPLOY_TARGET } from '../../drivers/workers/WorkerDriver.js';
import {
EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_DEPLOYS_PER_HOUR,
} from './limits.js';
import {
EventsWorkerDeployer,
LocalEventsInvokeTransport,
type EventsDeployOptions,
type EventsDeployOutcome,
} from './workerDeploy.js';
import {
eventsWorkerScript,
userHandlerScript,
} from '../../services/events/workerRuntime.js';
import {
appHandlerScope,
hashContent,
type HandlerScope,
} from '../../stores/events/EventHandlerStore.js';
import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
import { EventsWorkerDeployer } from './workerDeploy.js';
import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
import { handlerSetHash } from '../../services/events/workerSource.js';
import type { IConfig } from '../../types.js';
@@ -72,22 +46,12 @@ interface Owner {
type Layers = ConstructorParameters<typeof EventsWorkerDeployer>[0];
/** Stub layers whose only real dependency is a fresh redis mock. */
const makeLayers = (
owners: Record<string, Owner | null>,
events: Record<string, unknown> = {},
) => {
const makeLayers = (owners: Record<string, Owner | null>) => {
const redis = new MockRedis.Cluster(['redis://localhost:7001']);
const created: string[] = [];
const targets: Array<Record<string, unknown>> = [];
/** Apps whose handler sources were read, once per read. */
const sourceReads: string[] = [];
const layers = {
config: {
events: {
workerRuntime: true,
internalSecret: 'sekret',
...events,
},
events: { workerRuntime: true, internalSecret: 'sekret' },
// `#namespace()` needs a local-worker shape to resolve without
// throwing — this stub never reaches a real Cloudflare or
// Miniflare deploy either way.
@@ -96,20 +60,16 @@ const makeLayers = (
clients: { redis },
stores: {
eventHandler: {
setForApp: async () => [{ name: 'h', sourceHash: SOURCE_HASH }],
allForApp: async ({ appUid }: { appUid: string }) => {
sourceReads.push(appUid);
return [
{
appUid,
name: 'h',
source: 'async () => {}',
sourceHash: SOURCE_HASH,
createdAt: 0,
updatedAt: 0,
},
];
},
allForApp: async (appUid: string) => [
{
appUid,
name: 'h',
source: 'async () => {}',
sourceHash: SOURCE_HASH,
createdAt: 0,
updatedAt: 0,
},
],
},
app: {
getByUid: async (appUid: string) => {
@@ -120,18 +80,6 @@ const makeLayers = (
},
},
user: {
getByUuid: async (uuid: string) => {
const owner = Object.values(owners).find(
(o) => `uuid-${o?.ownerUserId}` === uuid,
);
if (!owner) return null;
return {
id: owner.ownerUserId,
uuid,
username: `user-${owner.ownerUserId}`,
suspended: owner.suspended ?? false,
};
},
getById: async (id: number) => {
const owner = Object.values(owners).find(
(o) => o?.ownerUserId === id,
@@ -149,24 +97,14 @@ const makeLayers = (
services: {},
drivers: {
workers: {
create: async (args: {
workerName: string;
[INTERNAL_DEPLOY_TARGET]?: Record<string, unknown>;
}) => {
create: async (args: { workerName: string }) => {
created.push(args.workerName);
targets.push(args[INTERNAL_DEPLOY_TARGET] ?? {});
return { success: true, errors: [] };
},
},
},
};
return {
layers: layers as unknown as Layers,
created,
targets,
redis,
sourceReads,
};
return { layers: layers as unknown as Layers, created, redis };
};
describe('EventsWorkerDeployer.ensure', () => {
@@ -182,17 +120,15 @@ describe('EventsWorkerDeployer.ensure', () => {
const deployer = new EventsWorkerDeployer(layers);
const [a, b] = await Promise.all([
deployer.ensure(appHandlerScope('app-a'), SCRIPT),
deployer.ensure(appHandlerScope('app-b'), SCRIPT),
deployer.ensure('app-a', SCRIPT),
deployer.ensure('app-b', SCRIPT),
]);
expect(a).toBe('deployed');
expect(b).toBe('no-owner');
});
it('refuses once the app has deployed past the hourly cap, before reading or building any source', async () => {
const { layers, redis, sourceReads, created } = makeLayers({
'app-c': { ownerUserId: 3 },
});
it('refuses once the app has deployed past the hourly cap', async () => {
const { layers, redis } = makeLayers({ 'app-c': { ownerUserId: 3 } });
const deployer = new EventsWorkerDeployer(layers);
const hour = new Date().toISOString().slice(0, 13);
await redis.set(
@@ -200,697 +136,12 @@ describe('EventsWorkerDeployer.ensure', () => {
String(EVENTS_WORKER_DEPLOYS_PER_HOUR),
);
expect(await deployer.ensure(appHandlerScope('app-c'), SCRIPT)).toBe(
'throttled',
);
expect(sourceReads).toEqual([]);
expect(created).toEqual([]);
});
it('answers a stale script without spending the hourly cap', async () => {
const { layers, redis, sourceReads } = makeLayers({
'app-e': { ownerUserId: 5 },
});
const deployer = new EventsWorkerDeployer(layers);
const stale = eventsWorkerScript(
handlerSetHash([{ name: 'h', sourceHash: 'b'.repeat(64) }]),
'',
);
expect(await deployer.ensure(appHandlerScope('app-e'), stale)).toBe(
'stale',
);
expect(sourceReads).toEqual([]);
const hour = new Date().toISOString().slice(0, 13);
expect(await redis.get(`ev:deploys:{app-e}:${hour}`)).toBeNull();
});
it('passes events.workerCpuMs to the deploy target', async () => {
const { layers, targets } = makeLayers(
{ 'app-f': { ownerUserId: 6 } },
{ workerCpuMs: 50 },
);
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.ensure(appHandlerScope('app-f'), SCRIPT)).toBe(
'deployed',
);
expect(targets[0]!.cpuMs).toBe(50);
});
it('passes no cpuMs when events.workerCpuMs is unset', async () => {
const { layers, targets } = makeLayers({ 'app-g': { ownerUserId: 7 } });
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.ensure(appHandlerScope('app-g'), SCRIPT)).toBe(
'deployed',
);
expect(targets[0]!.cpuMs).toBeUndefined();
expect(await deployer.ensure('app-c', SCRIPT)).toBe('throttled');
});
it('deploys normally under the cap', async () => {
const { layers } = makeLayers({ 'app-d': { ownerUserId: 4 } });
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.ensure(appHandlerScope('app-d'), SCRIPT)).toBe(
'deployed',
);
});
});
/** A server's schedules with a deploy gate that records and can refuse. */
const withGate = (
layers: Layers,
refuse: (capped: boolean) => boolean = () => false,
) => {
const gated: Array<{ script: string; capped: boolean }> = [];
const gate = {
gateDeploy: async (
script: string,
options: { capped: boolean },
deploy: () => Promise<EventsDeployOutcome>,
): Promise<EventsDeployOutcome> => {
gated.push({ script, capped: options.capped });
return refuse(options.capped) ? 'busy' : deploy();
},
};
const deployer = new EventsWorkerDeployer({
...layers,
services: { ...layers.services, schedules: gate },
} as unknown as Layers);
return { deployer, gated };
};
describe('the schedules` deploy gate', () => {
it('is skipped without schedules, or with schedules that have no gate', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
await expect(
new EventsWorkerDeployer(layers).ensure(
appHandlerScope('app-x'),
SCRIPT,
{
lane: 'schedule',
},
),
).resolves.toBe('deployed');
const gateless = new EventsWorkerDeployer({
...layers,
services: { ...layers.services, schedules: {} },
} as unknown as Layers);
await expect(
gateless.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
}),
).resolves.toBe('deployed');
expect(created).toEqual([SCRIPT, SCRIPT]);
});
it('takes every deploy, capped only for a schedule run', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer, gated } = withGate(layers);
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
).resolves.toBe('deployed');
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
}),
).resolves.toBe('deployed');
expect(gated).toEqual([
{ script: SCRIPT, capped: false },
{ script: SCRIPT, capped: true },
]);
expect(created).toEqual([SCRIPT, SCRIPT]);
});
it('answers a refused schedule run busy, and a delivery still deploys', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer } = withGate(layers, (capped) => capped);
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
}),
).resolves.toBe('busy');
await expect(
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
).resolves.toBe('deployed');
expect(created).toEqual([SCRIPT]);
});
it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer, gated } = withGate(layers, (capped) => capped);
const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
});
const deliveries = [
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
deployer.ensure(appHandlerScope('app-x'), SCRIPT),
];
await expect(scheduled).resolves.toBe('busy');
await expect(Promise.all(deliveries)).resolves.toEqual([
'deployed',
'deployed',
]);
expect(created).toEqual([SCRIPT]);
expect(gated.map((g) => g.capped)).toEqual([true, false]);
});
it('lets a schedule run join a delivery`s deploy in flight', async () => {
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
const { deployer, gated } = withGate(layers, (capped) => capped);
const delivery = deployer.ensure(appHandlerScope('app-x'), SCRIPT);
const scheduled = deployer.ensure(appHandlerScope('app-x'), SCRIPT, {
lane: 'schedule',
});
await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
'deployed',
'deployed',
]);
expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
expect(created).toEqual([SCRIPT]);
});
});
describe('LocalEventsInvokeTransport', () => {
it('renders each request as it goes out, after a cold deploy', async () => {
const bodies: string[] = [];
let resident = false;
const transport = new LocalEventsInvokeTransport(
{
localworkerservice: {
dispatchEventsWorker: async (request: Request) => {
bodies.push(await request.text());
if (!resident) return null;
return new Response(null, {
status: 200,
headers: { 'x-puter-events-handled': '1' },
});
},
},
} as never,
{
ensure: async () => {
resident = true;
return 'deployed';
},
scopeFor: async (appUid: string) => appHandlerScope(appUid),
} as unknown as EventsWorkerDeployer,
);
let renders = 0;
const call: EventsInvokeCall = {
script: SCRIPT,
appUid: 'app-x',
key: 'k',
body: '{"n":0}',
timeoutMs: 5000,
render: () => `{"n":${++renders}}`,
};
await expect(transport.send(call)).resolves.toMatchObject({
status: 200,
});
expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
});
});
describe('deploy-on-miss of an evicted script', () => {
it('redeploys the evicted script, then resends the run with the deployed marker', async () => {
const { layers, created } = makeLayers({ 'app-m': { ownerUserId: 8 } });
// The namespace only has the script once the deploy lands: before
// that the dispatcher answers a miss.
let resident = false;
layers.drivers.workers.create = (async (args: {
workerName: string;
}) => {
created.push(args.workerName);
resident = true;
return { success: true, errors: [] };
}) as never;
const deployer = new EventsWorkerDeployer(layers);
const sent: boolean[] = [];
const fetchImpl = (async (_url: string, init?: RequestInit) => {
const headers = (init?.headers ?? {}) as Record<string, string>;
sent.push(headers[EVENTS_DEPLOYED_HEADER] === '1');
if (resident)
return new Response(null, {
status: 200,
headers: { [EVENTS_HANDLED_HEADER]: '1' },
});
return new Response(null, {
status: 404,
headers: { [EVENTS_DISPATCH_ERROR_HEADER]: 'missing' },
});
}) as unknown as typeof undiciFetch;
const client = new EventsWorkerInvokerClient({
events: { workerRuntime: true },
} as IConfig);
client.setTransport(
new DispatcherInvokeTransport('http://dispatcher', 's', {
fetchImpl,
}),
);
client.setMissHandler((appUid, script, options) =>
deployer.ensure(appHandlerScope(appUid), script, options),
);
const result = await client.invoke({
script: SCRIPT,
appUid: 'app-m',
handler: 'h',
token: 't',
key: 'k',
event: {},
ctx: {},
});
expect(result).toEqual({ outcome: 'settled', status: 200 });
expect(created).toEqual([SCRIPT]);
// One miss, then the resend carrying the deployed marker.
expect(sent).toEqual([false, true]);
});
});
// The redis mock is shared across tests, so each one uses its own app and code.
let fresh = 0;
const nextApp = () => `app-u${++fresh}`;
const userScope = (holder: number, appUid: string) => ({ appUid, holder });
const sourceN = (label: string) =>
`async () => { await ${label}${++fresh}(); }`;
const scriptOf = (name: string, source: string) =>
userHandlerScript({ name, sourceHash: hashContent(source) }, '');
type Sets = Record<number, Array<{ name: string; source: string }>>;
/** Layers whose handler sets are per holder (0 is the app's own). */
const withSets = (layers: Layers, sets: Sets) => {
const handlersOf = (scope: { appUid: string; holder: number }) =>
(sets[scope.holder] ?? []).map((h) => ({
appUid: scope.appUid,
name: h.name,
source: h.source,
sourceHash: hashContent(h.source),
createdAt: 0,
updatedAt: 0,
}));
const store = layers.stores.eventHandler as unknown as Record<
string,
unknown
>;
store.setForApp = async (scope: { appUid: string; holder: number }) =>
handlersOf(scope).map(({ name, sourceHash }) => ({ name, sourceHash }));
store.allForApp = async (scope: { appUid: string; holder: number }) =>
handlersOf(scope);
};
const holders = (...ids: number[]) =>
Object.fromEntries(
ids.map((id) => [`holder-${id}`, { ownerUserId: id }]),
) as Record<string, Owner>;
describe('EventsWorkerDeployer in a user scope', () => {
it('deploys the one handler whose script was asked for, as the holder', async () => {
const app = nextApp();
const a = sourceN('a');
const b = sourceN('b');
const { layers, created, targets } = makeLayers(holders(10));
withSets(layers, {
10: [
{ name: 'a', source: a },
{ name: 'b', source: b },
],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('b', b);
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script]);
const target = targets[0]!;
expect(target.tags).toEqual([`events-app:${app}`, 'user:uuid-10']);
// One handler's script: the other one's code is not in it.
expect(target.source).toContain(b);
expect(target.source).not.toContain(a);
});
it('answers a script no handler of the scope matches as stale, before the throttle', async () => {
const app = nextApp();
const { layers, redis, created } = makeLayers(holders(10));
withSets(layers, { 10: [{ name: 'a', source: sourceN('a') }] });
const deployer = new EventsWorkerDeployer(layers);
expect(
await deployer.ensure(
userScope(10, app),
scriptOf('b', sourceN('b')),
),
).toBe('stale');
expect(created).toEqual([]);
const hour = new Date().toISOString().slice(0, 13);
expect(await redis.get(`ev:deploys:{${app}}:10:${hour}`)).toBeNull();
});
it('answers a scope with no handlers as no-handlers', async () => {
const { layers } = makeLayers(holders(10));
withSets(layers, {});
const deployer = new EventsWorkerDeployer(layers);
expect(
await deployer.ensure(
userScope(10, nextApp()),
scriptOf('a', sourceN('a')),
),
).toBe('no-handlers');
});
it('does not deploy for a holder that is suspended or gone', async () => {
const source = sourceN('a');
const { layers, created } = makeLayers({
'holder-10': { ownerUserId: 10, suspended: true },
});
withSets(layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
const app = nextApp();
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'no-owner',
);
expect(await deployer.ensure(userScope(11, app), script)).toBe(
'no-owner',
);
expect(created).toEqual([]);
});
it('gives each scope its own hourly cap, apart from the app scope', async () => {
const app = nextApp();
const source = sourceN('a');
const { layers, created } = makeLayers({
...holders(10, 11),
[app]: { ownerUserId: 12 },
});
withSets(layers, {
0: [{ name: 'a', source }],
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
for (let i = 0; i < EVENTS_USER_WORKER_DEPLOYS_PER_HOUR; i++)
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'deployed',
);
expect(await deployer.ensure(userScope(10, app), script)).toBe(
'throttled',
);
expect(created).toHaveLength(EVENTS_USER_WORKER_DEPLOYS_PER_HOUR);
// Another holder of the same app, and the app's own scope, still go.
expect(await deployer.ensure(userScope(11, app), script)).toBe(
'deployed',
);
expect(await deployer.ensure(appHandlerScope(app), script)).toBe(
'deployed',
);
});
it('collapses concurrent deploys of one script across holders into one upload', async () => {
const app = nextApp();
const source = sourceN('a');
const { layers, created } = makeLayers(holders(10, 11));
withSets(layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
await Promise.all([
deployer.ensure(userScope(10, app), script),
deployer.ensure(userScope(10, app), script),
deployer.ensure(userScope(11, app), script),
deployer.ensure(userScope(11, nextApp()), script),
]);
expect(created).toHaveLength(1);
});
it('names a script for identical code the same under any holder and app', async () => {
const source = sourceN('a');
const { layers, created } = makeLayers(holders(10, 11));
withSets(layers, {
10: [{ name: '@x', source }],
11: [{ name: '@x', source }],
});
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('@x', source);
expect(await deployer.ensure(userScope(10, nextApp()), script)).toBe(
'deployed',
);
expect(await deployer.ensure(userScope(11, nextApp()), script)).toBe(
'deployed',
);
expect(created).toEqual([script, script]);
});
});
describe('the deployed marker', () => {
const setup = () => {
const source = sourceN('a');
const made = makeLayers(holders(10, 11));
withSets(made.layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
return {
...made,
deployer: new EventsWorkerDeployer(made.layers),
script: scriptOf('a', source),
app: nextApp(),
};
};
it('is left by a user-scope deploy with a ttl under a day', async () => {
const { deployer, redis, script, app } = setup();
await deployer.ensure(userScope(10, app), script);
const ttl = await redis.ttl(`ev:deployed:${script}`);
expect(ttl).toBeGreaterThan(0);
expect(ttl).toBeLessThan(24 * 60 * 60);
});
it('lets an eager deploy skip the upload another holder already made', async () => {
const { deployer, created, script, app } = setup();
expect(await deployer.deployEagerly(userScope(10, app), script)).toBe(
'deployed',
);
expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script]);
});
it('spends no throttle on a skipped eager deploy', async () => {
const { deployer, redis, script, app } = setup();
await deployer.deployEagerly(userScope(10, app), script);
await deployer.deployEagerly(userScope(11, app), script);
const hour = new Date().toISOString().slice(0, 13);
expect(await redis.get(`ev:deploys:{${app}}:11:${hour}`)).toBeNull();
});
it('never skips a deploy-on-miss', async () => {
const { deployer, created, script, app } = setup();
await deployer.deployEagerly(userScope(10, app), script);
expect(await deployer.ensure(userScope(11, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script, script]);
});
it('is not left by an app-scope deploy', async () => {
const app = nextApp();
const source = sourceN('a');
const { layers, redis, created } = makeLayers({
[app]: { ownerUserId: 4 },
});
withSets(layers, { 0: [{ name: 'a', source }] });
const deployer = new EventsWorkerDeployer(layers);
const script = scriptOf('a', source);
await deployer.ensure(appHandlerScope(app), script);
expect(created).toEqual([script]);
expect(await redis.get(`ev:deployed:${script}`)).toBeNull();
});
it('still deploys eagerly when the marker cannot be read', async () => {
const { deployer, redis, created, script, app } = setup();
await deployer.deployEagerly(userScope(10, app), script);
const get = redis.get.bind(redis);
redis.get = (async (key: string) => {
if (key.startsWith('ev:deployed:')) throw new Error('down');
return get(key);
}) as never;
expect(await deployer.deployEagerly(userScope(11, app), script)).toBe(
'deployed',
);
expect(created).toEqual([script, script]);
});
});
describe('scopeFor', () => {
it('reads no holder as the app scope, and resolves a uuid to the user scope', async () => {
const { layers } = makeLayers(holders(10));
const deployer = new EventsWorkerDeployer(layers);
expect(await deployer.scopeFor('app-u')).toEqual({
appUid: 'app-u',
holder: 0,
});
expect(await deployer.scopeFor('app-u', '')).toEqual({
appUid: 'app-u',
holder: 0,
});
expect(await deployer.scopeFor('app-u', 'uuid-10')).toEqual({
appUid: 'app-u',
holder: 10,
});
expect(await deployer.scopeFor('app-u', 'uuid-99')).toBeNull();
});
});
describe('LocalEventsInvokeTransport in a user scope', () => {
it('deploys the holder`s script on a miss and refuses an unknown holder', async () => {
const source = sourceN('a');
const { layers, created } = makeLayers(holders(10));
withSets(layers, { 10: [{ name: 'a', source }] });
const deployer = new EventsWorkerDeployer(layers);
let resident = false;
const transport = new LocalEventsInvokeTransport(
{
localworkerservice: {
dispatchEventsWorker: async () => {
if (!resident) return null;
return new Response(null, {
status: 200,
headers: { 'x-puter-events-handled': '1' },
});
},
},
} as never,
{
ensure: async (
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
) => {
const outcome = await deployer.ensure(
scope,
script,
options,
);
resident = outcome === 'deployed';
return outcome;
},
scopeFor: (appUid: string, holder?: string) =>
deployer.scopeFor(appUid, holder),
} as unknown as EventsWorkerDeployer,
);
const script = scriptOf('a', source);
const call: EventsInvokeCall = {
script,
appUid: nextApp(),
holder: 'uuid-10',
key: 'k',
body: '{}',
timeoutMs: 5000,
};
await expect(transport.send(call)).resolves.toMatchObject({
status: 200,
});
expect(created).toEqual([script]);
resident = false;
await expect(
transport.send({ ...call, holder: 'uuid-99' }),
).resolves.toMatchObject({ status: null, error: 'deploy: no-holder' });
expect(created).toEqual([script]);
});
});
describe('concurrent eager deploys of one script', () => {
const setup = () => {
const source = sourceN('a');
const made = makeLayers(holders(10, 11));
withSets(made.layers, {
10: [{ name: 'a', source }],
11: [{ name: 'a', source }],
});
return {
...made,
script: scriptOf('a', source),
app: nextApp(),
};
};
it('upload once when two holders publish the same new code together', async () => {
const { layers, created, script, app } = setup();
const deployer = new EventsWorkerDeployer(layers);
const outcomes = await Promise.all([
deployer.deployEagerly(userScope(10, app), script),
deployer.deployEagerly(userScope(11, app), script),
]);
expect(outcomes).toEqual(['deployed', 'deployed']);
expect(created).toEqual([script]);
});
it('skip the upload when the marker appears while they wait on the gate', async () => {
const { layers, redis, created, script, app } = setup();
// Another node's deploy of the script finishes before the gate opens.
const gated = new EventsWorkerDeployer({
...layers,
services: {
schedules: {
gateDeploy: async (
_script: string,
_options: { capped: boolean },
deploy: () => Promise<EventsDeployOutcome>,
) => {
await redis.set(`ev:deployed:${script}`, '1');
return deploy();
},
},
},
} as unknown as Layers);
expect(
await gated.ensure(userScope(10, app), script, { eager: true }),
).toBe('deployed');
expect(created).toEqual([]);
// A miss still uploads, whatever the marker says.
expect(await gated.ensure(userScope(11, app), script)).toBe('deployed');
expect(created).toEqual([script]);
});
it('make a miss that joins an eager deploy deploy after it', async () => {
const { layers, redis, created, script, app } = setup();
const deployer = new EventsWorkerDeployer(layers);
await redis.set(`ev:deployed:${script}`, '1');
const eager = deployer.ensure(userScope(10, app), script, {
eager: true,
});
const miss = deployer.ensure(userScope(11, app), script);
expect(await eager).toBe('deployed');
expect(await miss).toBe('deployed');
// The eager one skipped on the marker; the miss uploaded.
expect(created).toEqual([script]);
expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
});
});
+66 -304
View File
@@ -26,100 +26,50 @@ import {
INTERNAL_ADMISSION_BYPASS,
INTERNAL_DEPLOY_TARGET,
} from '../../drivers/workers/WorkerDriver.js';
import type {
EventsInvokeAnswer,
EventsInvokeCall,
InvokeLane,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
import {
EVENTS_HANDLED_HEADER,
EVENTS_INVOKE_PATH,
answerFrom,
failedAnswer,
} from '../../clients/events/EventsWorkerInvokerClient.js';
import type { puterServices } from '../../services/index.js';
import type { IExtensionServiceInstances } from '../../services/types.js';
import {
eventsInvokeKey,
eventsWorkerScope,
eventsWorkerScript,
userHandlerScript,
} from '../../services/events/workerRuntime.js';
import {
generateEventsWorkerSource,
handlerSetHash,
} from '../../services/events/workerSource.js';
import {
appHandlerScope,
type HandlerScope,
} from '../../stores/events/EventHandlerStore.js';
import { generateEventsWorkerSource } from '../../services/events/workerSource.js';
import type { puterStores } from '../../stores/index.js';
import type { IConfig, LayerInstances } from '../../types.js';
import {
EVENTS_USER_WORKER_DEPLOYS_PER_HOUR,
EVENTS_WORKER_DEPLOYS_PER_HOUR,
} from './limits.js';
import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
/**
* Deploys an app's events worker from its published handlers.
*
* Deployed the first time a delivery needs it and again on eviction — a publish
* only writes rows (a user-scope publish also deploys eagerly). An app's scope
* is one script for the whole set; a user scope is one script per handler,
* deployed as the holder. Scripts carry no worker token and no `subdomains`
* row: not one of the owner's workers, not addressable from the internet, not
* listed or deleted as if it were. A republished set leaves the old script
* behind rather than deleting it; redeploying a still-live one is always safe.
* only writes rows. Scripts carry no worker token and no `subdomains` row: not
* one of the owner's workers, not addressable from the internet, not listed or
* deleted as if it were. A republished set leaves the old script behind rather
* than deleting it; redeploying a still-live one is always safe.
*/
/** Why a deploy could not happen, for the callers that answer differently. */
export type EventsDeployOutcome =
| 'deployed'
| 'stale'
| 'no-handlers'
| 'no-owner'
| 'throttled'
| 'failed'
| 'busy';
/** Whose call asked for the deploy: a schedule run's is held to a share. */
export interface EventsDeployOptions {
lane?: InvokeLane;
/**
* A user-scope publish's deploy, which skips the upload when the script's
* deployed marker is already set. A deploy-on-miss is never eager.
*/
eager?: boolean;
}
'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
interface DeployLayers {
config: IConfig;
clients: LayerInstances<typeof puterClients>;
stores: LayerInstances<typeof puterStores>;
services: LayerInstances<typeof puterServices> &
Pick<IExtensionServiceInstances, 'schedules'>;
services: LayerInstances<typeof puterServices>;
drivers: LayerInstances<typeof puterDrivers>;
}
/**
* `ev:deploys:{<appUid>}[:<holder>]:<hour>` bucket TTL — comfortably past the
* hour it counts.
* `ev:deploys:{<appUid>}:<hour>` bucket TTL — comfortably past the hour it
* counts.
*/
const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
/** How long `ev:deployed:<script>` says a script was uploaded. Under a day. */
const DEPLOYED_MARKER_TTL_SECONDS = 20 * 60 * 60;
const deployedMarkerKey = (script: string): string => `ev:deployed:${script}`;
const isUserScope = (scope: HandlerScope): boolean => scope.holder !== 0;
interface InFlightDeploy {
/** A schedule run's, which the region's deploy slots may refuse. */
capped: boolean;
eager: boolean;
promise: Promise<EventsDeployOutcome>;
}
interface DeployResult {
success?: boolean;
errors?: unknown[];
@@ -128,7 +78,7 @@ interface DeployResult {
export class EventsWorkerDeployer {
readonly #layers: DeployLayers;
/** Collapses the burst of misses one cold app produces into one deploy. */
readonly #inFlight = new Map<string, InFlightDeploy>();
readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
constructor(layers: DeployLayers) {
this.#layers = layers;
@@ -143,187 +93,39 @@ export class EventsWorkerDeployer {
}
/**
* Deploy `script`, which must be what the scope's handlers currently hash
* to: the whole set for an app, one handler for a user scope. A request for
* anything else is `stale` — the handlers moved on, and the next delivery
* resolves the new script rather than this one being revived. Always
* uploads; a deploy-on-miss means whatever said it was there is wrong.
* Deploy `script`, which must be what the app's handlers currently hash to.
* A request for anything else is `stale` — the set moved on, and the next
* delivery resolves the new script rather than this one being revived.
*/
ensure(
scope: HandlerScope,
script: string,
options: EventsDeployOptions = {},
): Promise<EventsDeployOutcome> {
// A user scope's script is the same bytes whoever holds it, so
// concurrent deploys of it from any holder or app are one.
const key = isUserScope(scope)
? `user|${script}`
: `${scope.appUid}|${scope.holder}|${script}`;
const capped = options.lane === 'schedule';
ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
const key = `${appUid}|${script}`;
const existing = this.#inFlight.get(key);
if (!existing) return this.#start(key, scope, script, options);
// An eager deploy may have skipped the upload, which a miss must not
// take for done.
if (existing.eager && !options.eager)
return this.#afterEager(
existing.promise,
key,
scope,
script,
options,
);
if (capped || !existing.capped) return existing.promise;
return this.#afterCapped(existing.promise, key, scope, script, options);
}
/**
* {@link ensure} for the deploy a user-scope publish starts, which skips the
* upload when another holder's deploy of the same script left its marker.
* Only this path reads the marker.
*/
async deployEagerly(
scope: HandlerScope,
script: string,
): Promise<EventsDeployOutcome> {
if (await this.#markedDeployed(script)) return 'deployed';
return this.ensure(scope, script, { eager: true });
}
/**
* The scope a script request names: the app's own, or the user scope of
* `holder` (a user uuid). Null for a holder that does not exist.
*/
async scopeFor(
appUid: string,
holder?: string | null,
): Promise<HandlerScope | null> {
if (!holder) return appHandlerScope(appUid);
const user = await this.#layers.stores.user.getByUuid(holder);
if (!user?.id) return null;
return { appUid, holder: Number(user.id) };
}
#start(
key: string,
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
const entry: InFlightDeploy = {
capped: options.lane === 'schedule',
eager: options.eager === true,
promise: this.#gated(scope, script, options).finally(() => {
if (this.#inFlight.get(key) === entry)
this.#inFlight.delete(key);
}),
};
this.#inFlight.set(key, entry);
return entry.promise;
}
/** A miss that found an eager deploy in flight deploys after it. */
async #afterEager(
eager: Promise<EventsDeployOutcome>,
key: string,
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
await eager;
const current = this.#inFlight.get(key);
if (current && !current.eager) return current.promise;
return this.#start(key, scope, script, options);
}
/**
* A delivery that joined a schedule run's deploy: `busy` means no deploy
* started, and a delivery is never refused a slot, so it asks again.
*/
async #afterCapped(
capped: Promise<EventsDeployOutcome>,
key: string,
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
const outcome = await capped;
if (outcome !== 'busy') return outcome;
const current = this.#inFlight.get(key);
if (current && !current.capped) return current.promise;
return this.#start(key, scope, script, options);
}
/**
* Every deploy goes through the schedules' deploy gate when a server has
* one: a per-script lock, so a burst of runs of an evicted script deploys
* it once, and a region-wide cap that a schedule run's deploy (`capped`)
* may be refused with `busy`. A delivery's deploy is never refused.
*/
#gated(
scope: HandlerScope,
script: string,
options: EventsDeployOptions,
): Promise<EventsDeployOutcome> {
const deploy = () =>
this.#deploy(scope, script, options.eager === true);
return (
this.#layers.services.schedules?.gateDeploy?.(
script,
{ capped: options.lane === 'schedule' },
deploy,
) ?? deploy()
if (existing) return existing;
const started = this.#deploy(appUid, script).finally(() =>
this.#inFlight.delete(key),
);
this.#inFlight.set(key, started);
return started;
}
async #deploy(
handlerScope: HandlerScope,
appUid: string,
script: string,
eager: boolean,
): Promise<EventsDeployOutcome> {
const { appUid } = handlerScope;
const { stores, drivers } = this.#layers;
const backend = eventsWorkerScope(this.#layers.config);
const userScope = isUserScope(handlerScope);
// Names and hashes are enough to tell a stale request, so the set's
// sources are read and built only once the throttle has let it through.
const set = await stores.eventHandler.setForApp(handlerScope);
if (set.length === 0) return 'no-handlers';
const handlers = await stores.eventHandler.allForApp(appUid);
if (handlers.length === 0) return 'no-handlers';
const generated = generateEventsWorkerSource(handlers);
// Two apps that published byte-identical sets share one script, which
// is sound: the source is the same, and everything per-app arrives with
// the invocation. Whoever deploys first owns it. A user scope's script
// is one handler's, so it is shared the same way across holders.
const wanted = userScope
? set.find((entry) => userHandlerScript(entry, backend) === script)
: undefined;
// the invocation. Whoever deploys first owns it.
if (
userScope
? !wanted
: eventsWorkerScript(handlerSetHash(set), backend) !== script
)
return 'stale';
// Asked again inside the gate, where another deploy of this script may
// have just finished.
if (eager && userScope && (await this.#markedDeployed(script)))
return 'deployed';
if (await this.#tooManyDeploysThisHour(handlerScope))
return 'throttled';
const all = await stores.eventHandler.allForApp(handlerScope);
const handlers = wanted
? all.filter(
(handler) =>
handler.name === wanted.name &&
handler.sourceHash === wanted.sourceHash,
)
: all;
const generated = generateEventsWorkerSource(handlers);
// The set can move between the two reads.
if (
handlers.length === 0 ||
eventsWorkerScript(generated.setHash, backend) !== script
eventsWorkerScript(
generated.setHash,
eventsWorkerScope(this.#layers.config),
) !== script
)
return 'stale';
if (generated.tooLarge) {
@@ -338,7 +140,9 @@ export class EventsWorkerDeployer {
generated.broken.join(', '),
);
const owner = await this.#owner(handlerScope);
if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
const owner = await this.#owner(appUid);
if (!owner) return 'no-owner';
const secret = this.#layers.config.events?.internalSecret;
@@ -367,7 +171,6 @@ export class EventsWorkerDeployer {
runtime: 'events',
source: generated.source,
omitOwnerToken: true,
cpuMs: this.#layers.config.events?.workerCpuMs,
// What the script compares an invocation against.
// Derived from the script name, so a redeploy of
// the same set binds the same key.
@@ -398,74 +201,34 @@ export class EventsWorkerDeployer {
);
return 'failed';
}
if (userScope) await this.#markDeployed(script);
return 'deployed';
}
async #markedDeployed(script: string): Promise<boolean> {
try {
return !!(await this.#layers.clients.redis.get(
deployedMarkerKey(script),
));
} catch (err) {
console.warn('[events] deployed marker unreadable', err);
return false;
}
}
async #markDeployed(script: string): Promise<void> {
try {
await this.#layers.clients.redis.set(
deployedMarkerKey(script),
'1',
'EX',
DEPLOYED_MARKER_TTL_SECONDS,
);
} catch (err) {
console.warn('[events] could not mark a script deployed', err);
}
}
/**
* Whether this scope has already deployed its hourly cap this hour. A
* flapping set (publish, remove, republish) provisions upstream on every
* miss, and this is the ceiling on that loop. An app's counter is per app;
* a user scope has its own, so one user cannot spend another's.
* Whether this app has already deployed `EVENTS_WORKER_DEPLOYS_PER_HOUR`
* times this hour. A flapping set (publish, remove, republish) provisions
* upstream on every miss, and this is the ceiling on that loop.
*/
async #tooManyDeploysThisHour(scope: HandlerScope): Promise<boolean> {
async #tooManyDeploysThisHour(appUid: string): Promise<boolean> {
const redis = this.#layers.clients.redis;
const hour = new Date().toISOString().slice(0, 13);
const userScope = isUserScope(scope);
const key = userScope
? `ev:deploys:{${scope.appUid}}:${scope.holder}:${hour}`
: `ev:deploys:{${scope.appUid}}:${hour}`;
const key = `ev:deploys:{${appUid}}:${hour}`;
const count = await redis.incr(key);
if (count === 1) await redis.expire(key, DEPLOY_THROTTLE_TTL_SECONDS);
return (
count >
(userScope
? EVENTS_USER_WORKER_DEPLOYS_PER_HOUR
: EVENTS_WORKER_DEPLOYS_PER_HOUR)
);
return count > EVENTS_WORKER_DEPLOYS_PER_HOUR;
}
/**
* The user a scope's deploy runs as, as an actor: the app's owner, or the
* holder of a user scope.
*/
/** The app owner, as an actor the deploy can run as. */
async #owner(
scope: HandlerScope,
appUid: string,
): Promise<
| (Actor & { user: { id: number; uuid: string; username: string } })
| null
> {
const { stores } = this.#layers;
let ownerUserId = scope.holder;
if (!isUserScope(scope)) {
const app = await stores.app.getByUid(scope.appUid);
ownerUserId = Number(app?.owner_user_id);
if (!app || !Number.isFinite(ownerUserId)) return null;
}
const app = await stores.app.getByUid(appUid);
const ownerUserId = Number(app?.owner_user_id);
if (!app || !Number.isFinite(ownerUserId)) return null;
const user = await stores.user.getById(ownerUserId);
if (!user?.id || !user.uuid || !user.username || user.suspended)
@@ -526,9 +289,11 @@ export class LocalEventsInvokeTransport {
this.#deployer = deployer;
}
async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
// Built per attempt, so a send time stamped into the body is taken
// after a cold deploy rather than before it.
async send(call: EventsInvokeCall): Promise<{
status: number | null;
handled?: boolean;
error?: string;
}> {
const request = (): Request =>
new Request(
`http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
@@ -538,7 +303,7 @@ export class LocalEventsInvokeTransport {
'content-type': 'application/json',
'x-puter-events-key': call.key,
},
body: call.render ? call.render() : call.body,
body: call.body,
},
);
@@ -546,27 +311,22 @@ export class LocalEventsInvokeTransport {
const resident = await this.#dispatch(request(), call.timeoutMs);
if (resident !== null) return resident;
const scope = await this.#deployer.scopeFor(
const outcome = await this.#deployer.ensure(
call.appUid,
call.holder,
call.script,
);
if (!scope) return { status: null, error: 'deploy: no-holder' };
const outcome = await this.#deployer.ensure(scope, call.script, {
lane: call.lane,
});
if (outcome !== 'deployed')
return {
status: null,
error: `deploy: ${outcome}`,
...(outcome === 'busy' ? { deployBusy: true } : {}),
};
return { status: null, error: `deploy: ${outcome}` };
const retried = await this.#dispatch(request(), call.timeoutMs);
if (retried === null)
return { status: null, error: 'script not resident' };
return retried;
} catch (err) {
return failedAnswer(err);
return {
status: null,
error: err instanceof Error ? err.message : String(err),
};
}
}
@@ -577,7 +337,7 @@ export class LocalEventsInvokeTransport {
async #dispatch(
request: Request,
timeoutMs: number,
): Promise<EventsInvokeAnswer | null> {
): Promise<{ status: number; handled: boolean } | null> {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<'timeout'>((resolve) => {
timer = setTimeout(() => resolve('timeout'), timeoutMs);
@@ -588,10 +348,12 @@ export class LocalEventsInvokeTransport {
this.#services.localworkerservice.dispatchEventsWorker(request),
timeout,
]);
if (response === 'timeout')
throw new DOMException('invocation timed out', 'TimeoutError');
if (response === 'timeout') throw new Error('invocation timed out');
if (response === null) return null;
return answerFrom(response);
return {
status: response.status,
handled: response.headers.get(EVENTS_HANDLED_HEADER) === '1',
};
} finally {
clearTimeout(timer);
}
@@ -42,7 +42,6 @@ import {
EVENTS_KV_HANDLES_PER_USER,
KV_SHARE_BATCH_MAX,
KV_SHARE_GRANT_LIMITS,
KV_SHARE_LOOKUP_LIMIT,
} from '../events/limits.js';
const BOOT_TIMEOUT_MS = 120_000;
@@ -68,7 +67,6 @@ const resetBudgets = async (env: PuterTestEnv) => {
const user = await env.server.stores.user.getByUsername(name);
for (const scope of [
EVENTS_KV_HANDLE_LIMIT.scope,
KV_SHARE_LOOKUP_LIMIT.scope,
...KV_SHARE_GRANT_LIMITS.map((window) => window.scope),
])
await env.server.clients.redis.del(`rate:${scope}:${user!.id}`);
@@ -125,7 +123,6 @@ describe('/kv/shares', () => {
'handle',
'holder',
'label',
'notify',
'prefix',
].sort(),
);
@@ -134,42 +131,6 @@ describe('/kv/shares', () => {
appUid: null,
holder: env.users.other.username,
label: 'Groceries',
notify: false,
});
});
it('takes notify, keeps it on a re-share that leaves it out, and shows it to both sides', async () => {
const prefix = `notify-${uuidv4().slice(0, 8)}:`;
const on = await shareWithOther({ prefix, notify: true });
expect(on).toMatchObject({ prefix, notify: true });
const kept = await shareWithOther({ prefix, access: 'write' });
expect(kept).toMatchObject({ handle: on.handle, notify: true });
const listed = async (path: string, token: string) =>
(
(await (await req('GET', path, token)).json()) as {
items: Array<Record<string, unknown>>;
}
).items.find((row) => row.handle === on.handle);
expect(
await listed('/kv/shares?limit=200', env.users.user.token),
).toMatchObject({ notify: true });
expect(
await listed('/kv/shares/with-me?limit=200', env.users.other.token),
).toMatchObject({ notify: true });
const off = await shareWithOther({ prefix, notify: false });
expect(off).toMatchObject({ handle: on.handle, notify: false });
const refused = await req('POST', '/kv/shares', env.users.user.token, {
prefix,
recipient: env.users.other.username,
notify: 'yes',
});
expect(refused.status).toBe(400);
await expect(refused.json()).resolves.toMatchObject({
code: 'kv_share_notify_invalid',
});
});
@@ -208,7 +169,6 @@ describe('/kv/shares', () => {
appUid: null,
holder: env.users.other.username,
label: 'Trip',
notify: false,
createdAt: shared.createdAt,
revokedAt: null,
revokedBy: null,
@@ -230,7 +190,6 @@ describe('/kv/shares', () => {
access: 'write',
appUid: null,
label: 'Trip',
notify: false,
createdAt: shared.createdAt,
});
expect(withMeText).not.toContain(env.users.user.username);
@@ -663,162 +622,6 @@ describe('/kv/shares', () => {
}
});
});
describe('sharing again', () => {
let ownerId: number;
let third: string;
beforeAll(async () => {
third = `kv-again-${uuidv4().slice(0, 8)}`;
await createTestUser(env.server, {
username: third,
password: 'pw-test-1234',
});
ownerId = (await env.server.stores.user.getByUsername(
env.users.user.username,
))!.id!;
});
const shareAt = (prefix: string, body: Record<string, unknown>) =>
req('POST', '/kv/shares', env.users.user.token, {
prefix,
...body,
});
/** Leaves `left` units in one of the owner's windows. */
const drain = (
window: { scope: string; limit: number; window: number },
left = 0,
) =>
claimRateLimit(
`${window.scope}:${ownerId}`,
window.limit,
window.window,
window.limit - left,
);
it('spends neither the mint nor the grant budget when every share stays as it is', async () => {
const prefix = `again-${uuidv4().slice(0, 8)}:`;
const recipients = [env.users.other.username, third];
const made = (await (
await shareAt(prefix, { recipients, label: 'Room' })
).json()) as { items: Array<{ handle: string }> };
await drain(EVENTS_KV_HANDLE_LIMIT);
for (const window of KV_SHARE_GRANT_LIMITS) await drain(window);
try {
const again = await shareAt(prefix, {
recipients: [...recipients, third],
});
expect(again.status).toBe(200);
const { items } = (await again.json()) as {
items: Array<{ handle: string; label: string }>;
};
expect(items.map((item) => item.handle)).toEqual([
made.items[0].handle,
made.items[1].handle,
made.items[1].handle,
]);
expect(items[0].label).toBe('Room');
const single = await shareAt(prefix, {
recipient: env.users.other.username,
access: 'read',
});
expect(single.status).toBe(200);
} finally {
await resetBudgets(env);
}
});
it('spends the mint budget on a change, and grant units only on recipients without the share', async () => {
const prefix = `again-${uuidv4().slice(0, 8)}:`;
const other = env.users.other.username;
expect((await shareAt(prefix, { recipient: other })).status).toBe(
200,
);
for (const window of KV_SHARE_GRANT_LIMITS) await drain(window);
try {
// A level change on a standing share takes no grant unit.
const raised = await shareAt(prefix, {
recipients: [other],
access: 'write',
});
expect(raised.status).toBe(200);
// Naming an unknown user next to it costs what a new share
// would, so the window refuses both.
for (const extra of ['nobody-by-this-name', third]) {
const refused = await shareAt(prefix, {
recipients: [other, extra],
access: 'write',
});
expect(refused.status).toBe(429);
}
} finally {
await resetBudgets(env);
}
await drain(EVENTS_KV_HANDLE_LIMIT);
try {
const lowered = await shareAt(prefix, {
recipient: other,
access: 'read',
});
expect(lowered.status).toBe(429);
await expect(lowered.json()).resolves.toMatchObject({
code: 'too_many_requests',
});
} finally {
await resetBudgets(env);
}
});
it('counts turning notify on or off as a change', async () => {
const prefix = `again-${uuidv4().slice(0, 8)}:`;
const other = env.users.other.username;
expect((await shareAt(prefix, { recipient: other })).status).toBe(
200,
);
await drain(EVENTS_KV_HANDLE_LIMIT);
try {
expect(
(await shareAt(prefix, { recipient: other })).status,
).toBe(200);
expect(
(
await shareAt(prefix, {
recipient: other,
notify: true,
})
).status,
).toBe(429);
} finally {
await resetBudgets(env);
}
});
it('spends the lookup budget on every recipient named', async () => {
const prefix = `again-${uuidv4().slice(0, 8)}:`;
const other = env.users.other.username;
expect((await shareAt(prefix, { recipient: other })).status).toBe(
200,
);
await resetBudgets(env);
await drain(KV_SHARE_LOOKUP_LIMIT, 1);
try {
expect(
(await shareAt(prefix, { recipient: other })).status,
).toBe(200);
const refused = await shareAt(prefix, { recipient: other });
expect(refused.status).toBe(429);
await expect(refused.json()).resolves.toMatchObject({
code: 'too_many_requests',
});
} finally {
await resetBudgets(env);
}
});
});
});
describe('/events/kv-handles beside /kv/shares', () => {
@@ -69,7 +69,6 @@ const shareBody = (share: KvShareView) => ({
appUid: publicAppUid(share.appUid),
holder: share.holderUsername,
label: share.label,
notify: share.notify,
createdAt: share.createdAt,
});
@@ -129,8 +128,8 @@ export class KvShareController extends PuterController {
/**
* POST /kv/shares — share a region with another account, `read` unless the
* body asks otherwise. Sharing the same region with the same account again
* changes its level, label or `notify` flag in place. `recipients` shares
* it with several at once and reports each one on its own item.
* changes its level or label in place. `recipients` shares it with several
* at once and reports each one on its own item.
*/
@Post('/shares', {
subdomain: 'api',
@@ -146,7 +145,6 @@ export class KvShareController extends PuterController {
access: body.access,
appUid: body.appUid,
label: body.label,
notify: body.notify,
};
const recipients = batchRecipients(body);
if (recipients) {
@@ -208,7 +206,6 @@ export class KvShareController extends PuterController {
appUid: publicAppUid(row.appUid),
holder: row.holderUsername,
label: row.label,
notify: row.notify,
createdAt: row.createdAt,
revokedAt: row.revokedAt,
revokedBy: row.revokedBy,
@@ -237,7 +234,6 @@ export class KvShareController extends PuterController {
access: row.access,
appUid: publicAppUid(row.appUid),
label: row.label,
notify: row.notify,
createdAt: row.createdAt,
})),
...(page.cursor ? { cursor: page.cursor } : {}),
@@ -21,9 +21,9 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
/**
* Route-level coverage for the notification endpoints the desktop calls over
* HTTP, so this suite is what catches a route going missing or a gate turning
* a legitimate dismiss or removal away.
* Route-level coverage for the notification endpoint. Dismissing is the only
* mark the desktop performs over HTTP, so this suite is what catches the route
* going missing or a gate turning a legitimate dismiss away.
*/
describe('notification endpoints over HTTP', () => {
let env: PuterTestEnv;
@@ -85,34 +85,4 @@ describe('notification endpoints over HTTP', () => {
});
expect(after?.acknowledged).toBeFalsy();
});
it('removes a notification', async () => {
const owner = env.users.user;
const { userId, uid } = await seedNotification(owner.username);
const res = await post('/notif/delete', owner.token, { uid });
expect(res.status).toBe(200);
await expect(res.json()).resolves.toEqual({});
const after = await env.server.stores.notification.getByUid(uid, {
userId,
});
expect(after).toBeNull();
});
it('refuses an anonymous removal', async () => {
const { userId, uid } = await seedNotification(env.users.other.username);
const res = await fetch(new URL('/notif/delete', env.apiOrigin), {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ uid }),
});
expect(res.status).toBeGreaterThanOrEqual(400);
const after = await env.server.stores.notification.getByUid(uid, {
userId,
});
expect(after?.uid).toBe(uid);
});
});
@@ -183,74 +183,3 @@ describe('NotificationController.markAck', () => {
expect(after?.acknowledged).toBeFalsy();
});
});
// ── /notif/delete ───────────────────────────────────────────────────
describe('NotificationController.remove', () => {
it('deletes the underlying notification row', async () => {
const { actor, userId } = await makeUser();
const created = await server.stores.notification.create({
userId,
value: { title: 't' },
});
const { res, captured } = makeRes();
await controller.remove(
makeReq({ body: { uid: created.uid }, actor }),
res,
);
expect(captured.body).toEqual({});
const after = await server.stores.notification.getByUid(
created.uid as string,
{ userId },
);
expect(after).toBeNull();
});
it('succeeds for a uid that names no row', async () => {
const { actor } = await makeUser();
const { res, captured } = makeRes();
await controller.remove(
makeReq({ body: { uid: 'no-such-uid' }, actor }),
res,
);
expect(captured.body).toEqual({});
});
it('rejects a missing uid with 400', async () => {
const { actor } = await makeUser();
const { res } = makeRes();
await expect(
controller.remove(makeReq({ body: {}, actor }), res),
).rejects.toMatchObject({ statusCode: 400 });
});
it('throws 401 when there is no actor on the request', async () => {
const { res } = makeRes();
await expect(
controller.remove(makeReq({ body: { uid: 'whatever' } }), res),
).rejects.toMatchObject({ statusCode: 401 });
});
it("does not delete another user's notification", async () => {
const a = await makeUser();
const b = await makeUser();
const created = await server.stores.notification.create({
userId: a.userId,
value: {},
});
const { res } = makeRes();
await controller.remove(
makeReq({ body: { uid: created.uid }, actor: b.actor }),
res,
);
const after = await server.stores.notification.getByUid(
created.uid as string,
{ userId: a.userId },
);
expect(after?.uid).toBe(created.uid);
});
});
@@ -20,44 +20,16 @@
import type { Request, Response } from 'express';
import { Controller, Post } from '../../core/http/decorators.js';
import { HttpError } from '../../core/http/HttpError.js';
import type { NotificationService } from '../../services/notification/NotificationService.js';
import { PuterController } from '../types.js';
/** Both routes fire per interaction; the ceiling catches a loop, not a user. */
const MARK_ROUTE_OPTIONS = {
subdomain: 'api',
requireUserActor: true,
allowFullAccessToken: true,
rateLimit: {
scope: 'notification-mark',
limit: 300,
window: 60_000,
key: 'user',
},
} as const;
/** The uid and owner a mark names, or the 400/401 the request earns. */
const markTarget = (req: Request): { uid: string; userId: number } => {
const uid = req.body?.uid;
if (typeof uid !== 'string' || uid.length === 0) {
throw new HttpError(400, '`uid` must be a non-empty string', {
legacyCode: 'bad_request',
});
}
const userId = req.actor?.user?.id;
if (!userId)
throw new HttpError(401, 'Unauthorized', {
legacyCode: 'unauthorized',
});
return { uid, userId };
};
/**
* GUI-facing notification endpoints. Supplement the `puter-notifications`
* driver (which handles reads via `/drivers/call`) with the mutations the puter
* desktop client calls directly.
* GUI-facing notification endpoint. Supplements the `puter-notifications`
* driver (which handles CRUD via `/drivers/call`) with the one mutation route
* the puter desktop client calls directly.
*
* Each goes through the NotificationService, which tells the user's other open
* tabs (`notif.ack`, `notif.removed`) so they update immediately.
* It emits `outer.gui.notif.ack` via the NotificationService so other open tabs
* for the same user see the state change immediately.
*/
@Controller('/notif')
export class NotificationController extends PuterController {
@@ -65,21 +37,50 @@ export class NotificationController extends PuterController {
* POST /notif/mark-ack — user dismissed a notification. Sets `acknowledged`
* timestamp; pushes ack event to sockets.
*/
@Post('/mark-ack', MARK_ROUTE_OPTIONS)
@Post('/mark-ack', {
subdomain: 'api',
requireUserActor: true,
allowFullAccessToken: true,
// Fires per notification interaction, so the ceiling stays
// generous — it is here to catch a loop, not to pace a user.
rateLimit: {
scope: 'notification-mark',
limit: 300,
window: 60_000,
key: 'user',
},
})
async markAck(req: Request, res: Response): Promise<void> {
const { uid, userId } = markTarget(req);
await this.services.notification.markAcknowledged(uid, userId);
res.json({});
}
const uid = req.body?.uid;
if (typeof uid !== 'string' || uid.length === 0) {
throw new HttpError(400, '`uid` must be a non-empty string', {
legacyCode: 'bad_request',
});
}
const userId = req.actor?.user?.id;
if (!userId)
throw new HttpError(401, 'Unauthorized', {
legacyCode: 'unauthorized',
});
/**
* POST /notif/delete — user removed a notification from their history. A
* uid naming none of their rows succeeds too: it is gone either way.
*/
@Post('/delete', MARK_ROUTE_OPTIONS)
async remove(req: Request, res: Response): Promise<void> {
const { uid, userId } = markTarget(req);
await this.services.notification.remove(uid, userId);
const notifService = this.services.notification as unknown as
| NotificationService
| undefined;
if (notifService?.markAcknowledged) {
await notifService.markAcknowledged(uid, userId);
} else {
// Fallback: direct store call if service isn't wired
await (
this.stores as Record<string, unknown> as {
notification: {
markAcknowledged: (
uid: string,
userId: number,
) => Promise<boolean>;
};
}
).notification.markAcknowledged(uid, userId);
}
res.json({});
}
}
@@ -244,10 +244,7 @@ describe('OIDCController GET /auth/oidc/providers', () => {
await callRoute('get', '/auth/oidc/providers', makeReq({}), res);
// We configured a single `custom` provider; the live service
// walks the config and returns its id.
expect(captured.body).toEqual({
providers: ['custom'],
signupOidcOnly: false,
});
expect(captured.body).toEqual({ providers: ['custom'] });
});
});
@@ -396,12 +396,9 @@ export class OIDCController extends PuterController {
},
},
async (_req: Request, res: Response) => {
const [providers, policy] = await Promise.all([
this.services.oidc.getEnabledProviderIds(),
this.services.signup.signupPolicy(),
]);
// The signup window drops its email form when this is set.
res.json({ providers, signupOidcOnly: policy.oidcOnly });
const providers =
await this.services.oidc.getEnabledProviderIds();
res.json({ providers });
},
);
@@ -511,31 +511,6 @@ describe('PuterAIController.openaiChatCompletions', () => {
const completeArgs = completeSpy.mock.calls[0]![0];
expect(completeArgs.provider).toBe('openai-completion');
expect(completeArgs.fallback).toBeUndefined();
});
it('forwards fallback: false so the driver stops at the first route', async () => {
const completeSpy = stubChatComplete({
message: { role: 'assistant', content: 'hi there' },
finish_reason: 'stop',
usage: { prompt_tokens: 4, completion_tokens: 2 },
});
const { res } = makeRes();
await controller.openaiChatCompletions(
makeReq({
body: {
model: 'gpt-test',
provider: 'openai-completion',
fallback: false,
messages: [{ role: 'user', content: 'hi' }],
},
actor: makeUserActor(),
}),
res,
);
expect(completeSpy.mock.calls[0]![0].fallback).toBe(false);
});
it('still pins OpenAI when the caller names no model, so the default model does not move', async () => {
@@ -448,7 +448,6 @@ export class PuterAIController extends PuterController {
}
: {}),
...openaiCompatProvider(body),
...(body.fallback === false ? { fallback: false } : {}),
};
const result = await this.#complete(res, completeArgs);
@@ -546,7 +545,8 @@ export class PuterAIController extends PuterController {
}
} else if (ev.type === 'usage') {
usageDetails = ev.usageDetails as
UsageDetails | undefined;
| UsageDetails
| undefined;
finishReason = ev.finish_reason as string | undefined;
} else if (ev.type === 'error') {
res.write(
@@ -649,7 +649,6 @@ export class PuterAIController extends PuterController {
: {}),
...(finiteMaxTokens(body.max_tokens) ?? {}),
...openaiCompatProvider(body),
...(body.fallback === false ? { fallback: false } : {}),
};
const completionId = `cmpl-${randomId()}`;
@@ -695,7 +694,8 @@ export class PuterAIController extends PuterController {
sendChunk(ev.text);
} else if (ev.type === 'usage') {
usageDetails = ev.usageDetails as
UsageDetails | undefined;
| UsageDetails
| undefined;
finishReason = ev.finish_reason as string | undefined;
} else if (ev.type === 'error') {
res.write(
@@ -749,7 +749,8 @@ export class PuterAIController extends PuterController {
text: extractTextContent(
(
messageResult.message as
Record<string, unknown> | undefined
| Record<string, unknown>
| undefined
)?.content,
),
index: 0,
@@ -868,7 +869,6 @@ export class PuterAIController extends PuterController {
}
: {}),
provider: providerName,
...(body.fallback === false ? { fallback: false } : {}),
};
const responseId = generateId('resp');
@@ -324,18 +324,6 @@ describe('parseAnthropicRequest', () => {
expect(args.provider).toBe('claude');
expect(args.normalize).toBe(false);
expect(args.streamToolInput).toBe(true);
expect(args.fallback).toBeUndefined();
});
it('forwards fallback: false and nothing else for fallback', () => {
expect(
parseAnthropicRequest({ messages: [], fallback: false }, headers)
.fallback,
).toBe(false);
expect(
parseAnthropicRequest({ messages: [], fallback: 'no' }, headers)
.fallback,
).toBeUndefined();
});
it('maps thinking, tool_choice, stop_sequences, top_p/top_k', () => {
@@ -431,7 +431,6 @@ export const parseAnthropicRequest = (
: {}),
...(betas.length ? { anthropicBetas: betas } : {}),
provider: toStringOrEmpty(body.provider) || 'claude',
...(body.fallback === false ? { fallback: false } : {}),
};
// Silently ignored: metadata, service_tier, inference_geo, container,
// fallbacks, diagnostics.
@@ -19,9 +19,8 @@
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import type { AddressInfo } from 'node:net';
import path from 'node:path';
import express, { type Request, type Response } from 'express';
import type { Request, Response } from 'express';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { PuterRouter } from '../../core/http/PuterRouter.js';
import type { RouteDescriptor } from '../../core/http/types.js';
@@ -112,7 +111,6 @@ const callGetHandler = async (
let captured: { filename?: string; root?: string } = {};
const req = {} as Request;
const res = {
setHeader: vi.fn(),
sendFile: vi.fn((filename: string, opts: { root: string }) => {
captured = { filename, root: opts.root };
}),
@@ -240,48 +238,6 @@ describe('StaticAssetsController puterjs_root', () => {
});
});
// ── push-sw.js caching ──────────────────────────────────────────────
/** Serves the route's handler over real HTTP and returns the response. */
const fetchThroughExpress = async (router: PuterRouter, routePath: string) => {
const route = findRoute(router, 'get', routePath);
if (!route) throw new Error(`No GET ${routePath} registered`);
const app = express();
app.get(routePath, (req, res, next) => route.handler(req, res, next));
const listener = app.listen(0);
await new Promise((resolve) => listener.once('listening', resolve));
try {
const { port } = listener.address() as AddressInfo;
return await fetch(`http://127.0.0.1:${port}${routePath}`);
} finally {
listener.close();
}
};
describe('StaticAssetsController push-sw.js', () => {
it.each([
['client_libs_root', 'puter.js/push-sw.js'],
['puterjs_root', 'push-sw.js'],
] as const)(
'lets the CDN keep it for five minutes from %s',
async (configKey, file) => {
const root = makeTempDir();
mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
writeFileSync(path.join(root, file), '/* worker */');
const router = buildController({ [configKey]: root });
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
const response = await fetchThroughExpress(router, route);
expect(response.status).toBe(200);
expect(response.headers.get('cache-control')).toBe(
'public, max-age=0, s-maxage=300',
);
expect(await response.text()).toBe('/* worker */');
}
},
);
});
// ── gui_assets_root ─────────────────────────────────────────────────
describe('StaticAssetsController gui_assets_root', () => {
@@ -23,9 +23,6 @@ import path from 'node:path';
import { PuterController } from '../types.js';
import type { PuterRouter } from '../../core/http/PuterRouter';
// Sites import push-sw.js by URL, so the CDN refetches it within five minutes.
const PUSH_WORKER_CACHE_CONTROL = 'public, max-age=0, s-maxage=300';
/**
* Static asset routes.
*
@@ -61,7 +58,6 @@ export class StaticAssetsController extends PuterController {
// `/v2/push-sw.js` is the path older docs gave.
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
router.get(route, { subdomain: 'js' }, (_req, res) => {
res.setHeader('Cache-Control', PUSH_WORKER_CACHE_CONTROL);
res.sendFile('puter.js/push-sw.js', { root });
});
}
@@ -112,10 +108,6 @@ export class StaticAssetsController extends PuterController {
});
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
router.get(route, { subdomain: 'js' }, (_req, res) => {
res.setHeader(
'Cache-Control',
PUSH_WORKER_CACHE_CONTROL,
);
res.sendFile('push-sw.js', { root: puterjsRoot });
});
}
-6
View File
@@ -87,11 +87,6 @@ export interface KnownContextFields {
* different provider's translation of it.
*/
strictUpstreamErrors: boolean;
/**
* Set while Puter files something into an account for it (mail delivery),
* so filesystem events never report that write as the request actor's own.
*/
platformWrite: boolean;
}
// Every key of `KnownContextFields`, checked both ways by the compiler, so
@@ -105,7 +100,6 @@ const KNOWN_KEY_FLAGS: Record<keyof KnownContextFields, true> = {
keepDriverSlot: true,
upstreamSignal: true,
strictUpstreamErrors: true,
platformWrite: true,
};
const KNOWN_KEYS: ReadonlySet<string> = new Set(Object.keys(KNOWN_KEY_FLAGS));
@@ -836,34 +836,6 @@ describe('requireVerifiedAccount', () => {
expectHttpError(got, 403, 'card_verification_required');
});
it('asks an email-gate-last account for card, then phone, then email', () => {
const pending = {
uuid: 'u-1',
requires_email_confirmation: true,
email_confirmed: false,
requires_phone_verification: true,
requires_card_verification: true,
metadata: { emailGateLast: true },
};
const gateFor = (user: Record<string, unknown>) =>
runGate(requireVerifiedAccount(), { actor: { user } });
expectHttpError(gateFor(pending), 403, 'card_verification_required');
expectHttpError(
gateFor({ ...pending, requires_card_verification: false }),
403,
'phone_verification_required',
);
expectHttpError(
gateFor({
...pending,
requires_card_verification: false,
requires_phone_verification: false,
}),
403,
'email_confirmation_required',
);
});
it('returns 403 two_factor_required for a seat whose team requires 2FA', async () => {
const teams = { getOrgSeat: async () => ({ require_2fa: 1 }) };
const got = await runGateAsync(requireTeam2fa(teams), {
+10 -22
View File
@@ -32,7 +32,6 @@ import {
type CardFallbackDeps,
} from '../../../util/cardFallback';
import { reserveUsernames } from '../../../util/reservedUsernames';
import { isEmailGateLast } from '../../../util/verificationOrder';
import {
isAccountContext,
isAppActor,
@@ -310,25 +309,11 @@ export const requireVerifiedAccount = (): RequestHandler =>
export const assertVerifiedAccount = (
user: AccountGateUser | undefined,
): void => {
const assertEmail = () => {
if (user?.requires_email_confirmation && !user?.email_confirmed) {
throw new HttpError(403, 'Please confirm your email to continue', {
legacyCode: 'email_confirmation_required',
});
}
};
const assertCard = () => {
if (user?.requires_card_verification) {
throw new HttpError(403, 'Please verify your card to continue', {
legacyCode: 'card_verification_required',
});
}
};
// Email → phone → card, or card → phone → email for an email-gate-last
// account, so the client is sent to the gate it has to clear next.
const emailLast = isEmailGateLast(user);
if (emailLast) assertCard();
else assertEmail();
if (user?.requires_email_confirmation && !user?.email_confirmed) {
throw new HttpError(403, 'Please confirm your email to continue', {
legacyCode: 'email_confirmation_required',
});
}
if (user?.requires_phone_verification) {
throw new HttpError(
403,
@@ -338,8 +323,11 @@ export const assertVerifiedAccount = (
},
);
}
if (emailLast) assertEmail();
else assertCard();
if (user?.requires_card_verification) {
throw new HttpError(403, 'Please verify your card to continue', {
legacyCode: 'card_verification_required',
});
}
// A team seat signs in on a password its administrator still holds,
// so it reaches nothing until it has replaced that password.
if (user?.requires_password_change) {
@@ -1,163 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { EventEmitter } from 'node:events';
import type { NextFunction, Request, RequestHandler, Response } from 'express';
import { describe, expect, it, vi } from 'vitest';
import { isHttpError } from '../HttpError';
import {
createLargeJsonBodyParser,
DEFAULT_JSON_LIMIT_BYTES,
} from './largeJsonBody';
const MiB = 1024 * 1024;
// A complete request with no socket reads as already consumed to body-parser,
// so the parsers call straight through, leaving any preset `body` in place.
const makeReq = (
bytes: number | undefined,
user: string | null = 'u1',
body?: unknown,
) =>
({
complete: true,
body,
headers: {
'content-type': 'application/json',
...(bytes === undefined ? {} : { 'content-length': String(bytes) }),
...(user ? { authorization: user } : {}),
},
}) as unknown as Request;
const makeRes = () => {
const res = new EventEmitter() as EventEmitter & {
setHeader: ReturnType<typeof vi.fn>;
};
res.setHeader = vi.fn();
return res;
};
const authProbe = vi.fn<RequestHandler>((req, _res, next) => {
if (req.headers.authorization) {
(req as { actor?: unknown }).actor = {
user: { uuid: req.headers.authorization },
};
}
next();
});
const run = (mw: RequestHandler, req: Request, res = makeRes()) =>
new Promise<{ err: unknown; res: ReturnType<typeof makeRes> }>(
(resolve) => {
mw(
req,
res as unknown as Response,
((err?: unknown) => resolve({ err, res })) as NextFunction,
);
},
);
const statusOf = (err: unknown) => (isHttpError(err) ? err.statusCode : err);
describe('createLargeJsonBodyParser', () => {
const create = ({
budgetBytes = 200 * MiB,
perUserBytes = 128 * MiB,
accepts,
}: {
budgetBytes?: number;
perUserBytes?: number;
accepts?: (req: Request) => boolean;
} = {}) =>
createLargeJsonBodyParser({
authProbe,
limitBytes: 128 * MiB,
budgetBytes,
perUserBytes,
accepts,
});
it.each([
['no Content-Length', undefined],
['a body within the default limit', DEFAULT_JSON_LIMIT_BYTES],
['a body over the large limit', 128 * MiB + 1],
])('leaves %s to the global parser without probing', async (_, bytes) => {
authProbe.mockClear();
const { err } = await run(create(), makeReq(bytes));
expect(err).toBeUndefined();
expect(authProbe).not.toHaveBeenCalled();
});
it('leaves an unauthenticated large body to the global parser', async () => {
const mw = create({ budgetBytes: 60 * MiB });
const { err } = await run(mw, makeReq(60 * MiB, null));
expect(err).toBeUndefined();
// Nothing was charged, so a signed-in caller still fits.
expect((await run(mw, makeReq(60 * MiB))).err).toBeUndefined();
});
it('charges nothing for a client that left during the probe', async () => {
const mw = create({ budgetBytes: 60 * MiB });
const gone = Object.assign(makeRes(), { closed: true });
expect((await run(mw, makeReq(60 * MiB), gone)).err).toBeUndefined();
expect((await run(mw, makeReq(60 * MiB))).err).toBeUndefined();
});
it('refuses with 503 past the budget until a response closes', async () => {
const mw = create({ budgetBytes: 130 * MiB });
const first = await run(mw, makeReq(60 * MiB, 'u1'));
await run(mw, makeReq(60 * MiB, 'u2'));
const refused = await run(mw, makeReq(60 * MiB, 'u3'));
expect(statusOf(refused.err)).toBe(503);
// Load shedding is recorded, not paged.
expect(isHttpError(refused.err) && refused.err.alarmSeverity).toBe(
'info',
);
expect(refused.res.setHeader).toHaveBeenCalledWith('Retry-After', '5');
first.res.emit('close');
expect((await run(mw, makeReq(60 * MiB, 'u3'))).err).toBeUndefined();
});
it("refuses one user's bodies with 429 past their share", async () => {
const mw = create({ perUserBytes: 100 * MiB });
const first = await run(mw, makeReq(60 * MiB, 'u1'));
const refused = await run(mw, makeReq(60 * MiB, 'u1'));
expect(statusOf(refused.err)).toBe(429);
expect(refused.res.setHeader).toHaveBeenCalledWith('Retry-After', '5');
// Another user isn't held back by it.
expect((await run(mw, makeReq(60 * MiB, 'u2'))).err).toBeUndefined();
first.res.emit('close');
expect((await run(mw, makeReq(60 * MiB, 'u1'))).err).toBeUndefined();
});
it('refuses with 413 a parsed body `accepts` turns down', async () => {
const mw = create({
accepts: (req) => (req.body as { ok?: boolean }).ok === true,
});
const refused = await run(mw, makeReq(60 * MiB, 'u1', { ok: false }));
expect(statusOf(refused.err)).toBe(413);
expect(
(await run(mw, makeReq(60 * MiB, 'u2', { ok: true }))).err,
).toBeUndefined();
});
});
@@ -1,134 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import express, { type Request, type RequestHandler } from 'express';
import { actorUid } from '../../actor';
import { HttpError } from '../HttpError';
/** The global JSON parser's limit, which applies before auth. */
export const DEFAULT_JSON_LIMIT_BYTES = 50 * 1024 * 1024;
interface LargeJsonBodyOptions {
/** The global auth probe; it skips requests that already carry an actor. */
authProbe: RequestHandler;
limitBytes: number;
/** Declared bytes of large bodies a process holds at once. */
budgetBytes: number;
/** Declared bytes of large bodies one user holds at once. */
perUserBytes: number;
/** Checked once the body is parsed; a body it refuses gets a 413. */
accepts?: (req: Request) => boolean;
}
/**
* Parses JSON bodies over the default limit, up to `limitBytes`, once the
* caller's header or cookie credential resolves. Unauthenticated requests and
* requests without a Content-Length fall through to the global parser and its
* limit.
*
* The parsed body lives as long as the request, so a body's bytes count against
* both budgets until its response closes. Over the user's budget gets a 429,
* over the process budget a 503.
*/
export const createLargeJsonBodyParser = ({
authProbe,
limitBytes,
budgetBytes,
perUserBytes,
accepts,
}: LargeJsonBodyOptions): RequestHandler => {
const json = express.json({ limit: limitBytes });
const textAsJson = express.json({
limit: limitBytes,
type: (req) =>
req.headers['content-type'] === 'text/plain;actually=json',
});
let inFlightBytes = 0;
const inFlightByUser = new Map<string, number>();
return (req, res, next) => {
const declared = Number(req.headers['content-length']);
if (!(declared > DEFAULT_JSON_LIMIT_BYTES && declared <= limitBytes)) {
next();
return;
}
void authProbe(req, res, () => {
// `res.closed`: the client left during the probe, so `close` has
// already fired and would never release a charge.
if (!req.actor || res.closed) {
next();
return;
}
const user = req.actor.user.uuid ?? actorUid(req.actor);
const userBytes = inFlightByUser.get(user) ?? 0;
if (userBytes + declared > perUserBytes) {
res.setHeader('Retry-After', '5');
next(
new HttpError(
429,
'Too many large requests in progress for this account',
{ legacyCode: 'too_many_requests' },
),
);
return;
}
if (inFlightBytes + declared > budgetBytes) {
res.setHeader('Retry-After', '5');
next(
new HttpError(503, 'Too many large requests in progress', {
code: 'large_body_capacity',
// Load shedding, not a fault: record it, don't page.
alarmSeverity: 'info',
}),
);
return;
}
inFlightBytes += declared;
inFlightByUser.set(user, userBytes + declared);
res.once('close', () => {
inFlightBytes -= declared;
const left = (inFlightByUser.get(user) ?? 0) - declared;
if (left > 0) inFlightByUser.set(user, left);
else inFlightByUser.delete(user);
});
json(req, res, (err?: unknown) => {
if (err) {
next(err);
return;
}
textAsJson(req, res, (err?: unknown) => {
if (err) {
next(err);
return;
}
if (accepts && !accepts(req)) {
// The same answer the default limit gives.
next(
new HttpError(413, 'request entity too large', {
legacyCode: 'client_bad_request',
}),
);
return;
}
next();
});
});
});
};
};
@@ -84,10 +84,6 @@ export function sweepMemoryWindows() {
)
memoryWindows.delete(k);
}
for (const [k, entry] of memoryCounters) {
if (entry.index < Math.floor(now / entry.windowMs) - 1)
memoryCounters.delete(k);
}
}
{
@@ -159,97 +155,6 @@ async function peekMemory(key, limit, windowMs) {
return timestamps.length < limit;
}
// -- Counted claims --------------------------------------------------
//
// `claim` keeps a member per unit, which is fine for a few units and not for
// thousands. The counted form keeps two counters per key, the current fixed
// window and the one before it, and slides by weighting the previous one by
// how much of it the window still covers. It can be off by a little at window
// edges, and its buckets are separate from the other forms'.
/** The sliding-window estimate from the two fixed windows it overlaps. */
const slidingCount = (previous, current, now, windowMs) =>
previous * (1 - (now % windowMs) / windowMs) + current;
/** How many of `units` a counted claim that left `used` spent may keep. */
const countedAdmits = (used, limit, units) =>
units - Math.min(units, Math.max(0, Math.ceil(used - limit)));
/** Key → `{ index, current, previous, windowMs }`, swept with the windows. */
const memoryCounters = new Map();
async function claimCountMemory(key, limit, windowMs, units) {
const now = Date.now();
const index = Math.floor(now / windowMs);
let entry = memoryCounters.get(key);
if (!entry) {
if (memoryCounters.size >= MEMORY_MAX_KEYS) {
memoryCounters.delete(memoryCounters.keys().next().value);
}
entry = { index, current: 0, previous: 0, windowMs };
memoryCounters.set(key, entry);
} else if (entry.index !== index) {
entry.previous = entry.index === index - 1 ? entry.current : 0;
entry.current = 0;
entry.index = index;
}
entry.windowMs = windowMs;
const used = slidingCount(entry.previous, entry.current, now, windowMs);
const admitted = countedAdmits(used + units, limit, units);
entry.current += admitted;
return admitted;
}
async function claimCountRedis(redis, key, limit, windowMs, units) {
const now = Date.now();
const index = Math.floor(now / windowMs);
// The hash tag keeps both windows in one slot, so one MULTI reads both.
const current = `ratec:{${key}}:${index}`;
const results = await redis
.multi()
.incrby(current, units)
.pexpire(current, 2 * windowMs)
.get(`ratec:{${key}}:${index - 1}`)
.exec();
const used = slidingCount(
Number(multiResult(results, 2) ?? 0),
Number(multiResult(results, 0)),
now,
windowMs,
);
const admitted = countedAdmits(used, limit, units);
if (admitted < units) await redis.decrby(current, units - admitted);
return admitted;
}
async function claimCountKv(kv, key, limit, windowMs, units) {
const now = Date.now();
const index = Math.floor(now / windowMs);
const current = `ratec:${key}:${index}`;
const [counted, previous] = await Promise.all([
kv.incr({
key: current,
pathAndAmountMap: { '': units },
expireAt: Math.ceil((now + 2 * windowMs) / 1000),
}),
kv.get({ key: `ratec:${key}:${index - 1}` }),
]);
const used = slidingCount(
Number(previous.res ?? 0),
Number(counted.res),
now,
windowMs,
);
const admitted = countedAdmits(used, limit, units);
if (admitted < units) {
await kv.decr({
key: current,
pathAndAmountMap: { '': units - admitted },
});
}
return admitted;
}
// -- Redis backend ---------------------------------------------------
// ioredis MULTI/EXEC reports per-command failures inside the exec() result
@@ -570,10 +475,6 @@ function instrumentBackendPair(name, pair) {
withSpan('rate_limit.claim', attrs, () =>
pair.claim(key, limit, windowMs, units),
),
count: (key, limit, windowMs, units) =>
withSpan('rate_limit.count', attrs, () =>
pair.count(key, limit, windowMs, units),
),
acquire: (key, limit) =>
withSpan('rate_limit.acquire', attrs, () =>
pair.acquire(key, limit),
@@ -585,7 +486,6 @@ const memoryBackendPair = instrumentBackendPair('memory', {
rate: checkMemory,
peek: peekMemory,
claim: claimMemory,
count: claimCountMemory,
acquire: acquireMemoryConcurrent,
});
@@ -636,8 +536,6 @@ export function configureRateLimit({
peekRedis(redis, key, limit, windowMs),
claim: (key, limit, windowMs, units) =>
claimRedis(redis, key, limit, windowMs, units),
count: (key, limit, windowMs, units) =>
claimCountRedis(redis, key, limit, windowMs, units),
acquire: (key, limit) => acquireRedisConcurrent(redis, key, limit),
});
}
@@ -647,8 +545,6 @@ export function configureRateLimit({
peek: (key, limit) => peekKv(kv, key, limit),
claim: (key, limit, windowMs, units) =>
claimKv(kv, key, limit, windowMs, units),
count: (key, limit, windowMs, units) =>
claimCountKv(kv, key, limit, windowMs, units),
acquire: (key, limit) => acquireKvConcurrent(kv, key, limit),
});
}
@@ -978,32 +874,6 @@ export async function claimRateLimit(key, limit, windowMs, units, backend) {
}
}
/**
* `claimRateLimit` for budgets measured in many units per call: a claim costs
* the same however many units it spends (see "Counted claims" above). Its
* buckets are its own, so a key is only ever claimed one way. Fails open,
* admitting every unit.
*/
export async function claimCountedRateLimit(
key,
limit,
windowMs,
units,
backend,
) {
if (!(units > 0)) return 0;
const bk = resolveBackend(backend);
try {
return await bk.count(key, limit, windowMs, units);
} catch (err) {
console.error(
'[rate-limit] imperative counted claim failed, failing open:',
err,
);
return units;
}
}
/**
* Charge a route-shaped rate-limit spec from inside a handler.
*
@@ -37,7 +37,6 @@ import {
acquireDriverConcurrent,
checkDriverRateLimit,
checkRateLimit,
claimCountedRateLimit,
claimRateLimit,
concurrencyGate,
consumeRouteRateLimit,
@@ -1488,13 +1487,6 @@ describe('rate + concurrent limiting — kv backend', () => {
expect(await claimRateLimit(key, 5, 60_000, 1)).toBe(0);
});
it('claims counted units, admitting only what fits', async () => {
const key = uniqueScope();
expect(await claimCountedRateLimit(key, 5, 60_000, 3)).toBe(3);
expect(await claimCountedRateLimit(key, 5, 60_000, 3)).toBe(2);
expect(await claimCountedRateLimit(key, 5, 60_000, 1)).toBe(0);
});
it('claims against a full bucket as full while expired rows are unswept', async () => {
const key = uniqueScope();
await seedUnswept(`rate:${key}:`, 3);
@@ -2018,103 +2010,6 @@ describe('acquireConcurrent (imperative)', () => {
});
});
describe('claimCountedRateLimit', () => {
const uniqueKey = () =>
`counted-${Math.random().toString(36).slice(2, 10)}`;
const WINDOW = 10_000;
/** The start of a fixed window, so nothing before it carries in. */
const windowStart = (n) => vi.setSystemTime(n * WINDOW);
afterEach(() => {
vi.useRealTimers();
});
describe.each([
['memory', () => configureRateLimit()],
[
'redis',
() =>
configureRateLimit({
default: 'redis',
redis: new RedisMock(),
}),
],
])('%s backend', (_name, configure) => {
beforeEach(() => {
configure();
vi.useFakeTimers({ toFake: ['Date'] });
windowStart(1_000);
});
it('admits up to the limit and spends only what fits', async () => {
const key = uniqueKey();
expect(await claimCountedRateLimit(key, 5, WINDOW, 3)).toBe(3);
expect(await claimCountedRateLimit(key, 5, WINDOW, 3)).toBe(2);
expect(await claimCountedRateLimit(key, 5, WINDOW, 1)).toBe(0);
});
it('spends thousands of units in one claim', async () => {
const key = uniqueKey();
expect(
await claimCountedRateLimit(key, 15_000, WINDOW, 6_000),
).toBe(6_000);
expect(
await claimCountedRateLimit(key, 15_000, WINDOW, 10_000),
).toBe(9_000);
});
it('carries the previous window by how much of it is still inside', async () => {
const key = uniqueKey();
expect(await claimCountedRateLimit(key, 100, WINDOW, 100)).toBe(
100,
);
// A quarter into the next window, 75 of those still count.
vi.setSystemTime(1_001 * WINDOW + WINDOW / 4);
expect(await claimCountedRateLimit(key, 100, WINDOW, 50)).toBe(25);
// Two windows on, nothing carries.
windowStart(1_003);
expect(await claimCountedRateLimit(key, 100, WINDOW, 100)).toBe(
100,
);
});
it('keeps its buckets apart from claimRateLimit', async () => {
const key = uniqueKey();
expect(await claimCountedRateLimit(key, 2, WINDOW, 2)).toBe(2);
expect(await claimRateLimit(key, 2, WINDOW, 2)).toBe(2);
});
});
it('claims nothing for zero units', async () => {
const key = uniqueKey();
expect(await claimCountedRateLimit(key, 1, WINDOW, 0)).toBe(0);
expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
});
it('fails open, admitting every unit, when the backend throws', async () => {
const redis = new RedisMock();
redis.multi = () => {
throw new Error('redis down');
};
configureRateLimit({ default: 'redis', redis });
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
expect(await claimCountedRateLimit(uniqueKey(), 1, WINDOW, 4)).toBe(4);
spy.mockRestore();
});
it('drops a memory counter once its windows have passed', async () => {
configureRateLimit();
vi.useFakeTimers({ toFake: ['Date'] });
windowStart(2_000);
const key = uniqueKey();
expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
windowStart(2_002);
sweepMemoryWindows();
windowStart(2_000);
expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
});
});
describe('claimRateLimit', () => {
const uniqueKey = () => `claim-${Math.random().toString(36).slice(2, 10)}`;
@@ -290,26 +290,6 @@ describe('createRouteLifecycleMiddleware', () => {
expect(after).toHaveLength(0);
});
it('answers 429 when the vetoing listener asks for it', async () => {
const events = makeEvents();
events.on(`${BASE}.before` as never, (_k, data) => {
(data as RouteLifecycleEvent).allow = false;
(data as RouteLifecycleEvent).rejectReason = 'slow down';
(data as RouteLifecycleEvent).rejectStatus = 429;
});
const reject = record(events, `${BASE}.reject` as keyof EventMap);
const mw = createRouteLifecycleMiddleware(events, POST, PATH);
const res = new FakeRes();
await mw(makeReq('u-9'), res as unknown as Response, () => {});
expect(res.statusCode).toBe(429);
expect(res.body).toMatchObject({
error: { code: 'too_many_requests', message: 'slow down' },
});
expect(reject[0]).toMatchObject({ statusCode: 429 });
});
it('treats a listener-sent response as a terminal after (not a reject) with the real status', async () => {
const events = makeEvents();
events.on(`${BASE}.before` as never, (_k, data) => {
+5 -9
View File
@@ -20,7 +20,6 @@
import type { Request, RequestHandler } from 'express';
import { actorUid } from '../actor';
import type { EventClient } from '../../clients/event/EventClient';
import type { LifecycleRejectStatus } from '../../clients/event/types';
import type { RouteMethod } from './types';
type RoutePathValue = string | RegExp | Array<string | RegExp> | undefined;
@@ -101,33 +100,30 @@ export const createRouteLifecycleMiddleware = (
...base,
allow: true as boolean,
rejectReason: undefined as string | undefined,
rejectStatus: undefined as LifecycleRejectStatus | undefined,
};
if (observeBefore) {
await events.emitAndWait(beforeKey, beforeEvent, {});
}
// Explicit veto: a listener set `allow = false`. Emit `reject` and
// answer 403 (or the listener's 429) — unless the listener already
// wrote its own response, in which case we report that status.
// answer 403 — unless the listener already wrote its own response, in
// which case we report that status rather than a misleading 403.
if (beforeEvent.allow === false) {
const status = beforeEvent.rejectStatus === 429 ? 429 : 403;
events.emit(
`${keyBase}.reject`,
{
phase: 'reject',
...base,
statusCode: res.headersSent ? res.statusCode : status,
statusCode: res.headersSent ? res.statusCode : 403,
durationMs: Date.now() - startedAt,
rejectReason: beforeEvent.rejectReason,
},
{},
);
if (!res.headersSent) {
res.status(status).json({
res.status(403).json({
error: {
code:
status === 429 ? 'too_many_requests' : 'forbidden',
code: 'forbidden',
message:
beforeEvent.rejectReason ?? 'Blocked by policy',
},
-1
View File
@@ -76,7 +76,6 @@ export type AccountGateUser = Partial<
| 'phone'
| 'card_fingerprint'
| 'otp_enabled'
| 'metadata'
>
> & { id?: number };
-42
View File
@@ -1,42 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { describe, expect, it } from 'vitest';
import { isHttpError } from './http/HttpError.js';
import {
SCHEDULE_LIST_LIMIT_CAP,
SCHEDULE_PEER_PATH,
schedulesUnavailable,
} from './schedules.js';
describe('schedules contract', () => {
it('answers an absent schedules service with a 503 events_disabled', () => {
const err = schedulesUnavailable();
expect(isHttpError(err)).toBe(true);
expect(err.statusCode).toBe(503);
expect(err.legacyCode).toBe('events_disabled');
// Expected on a server without schedules, so it must not page.
expect(err.noAlarm).toBe(true);
});
it('keeps the list cap and the peer path the extension relies on', () => {
expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
});
});
-198
View File
@@ -1,198 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* What core asks of schedules. Nothing here is an implementation: an extension
* registers `services.schedules`, and a server without one answers every
* schedule route and `notify({ sendAt })` with `events_disabled`. Called by
* extensions; keep these signatures stable.
*/
import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
import type { AppMessage } from '../services/notification/appMessage.js';
import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
import type { DurableSubscription } from '../stores/events/types.js';
import type { PageResult } from '../util/pagination.js';
import type { Actor } from './actor.js';
import { HttpError } from './http/HttpError.js';
/** Why a schedule ended. */
export type ScheduleEndReason =
| 'removed'
| 'ended'
| 'app_deleted'
| 'uninstalled'
| 'permission_revoked'
| 'holder_gone'
| 'app_gone'
| 'orphaned'
| 'delivered'
| 'dropped';
/** What happened to a set of subscription rows schedules own. */
export type ScheduleSubscriptionChange =
| {
type: 'suspended';
reason: SuspendedReason;
/** For a reason that never lifts; defaults to that reason. */
endReason?: ScheduleEndReason;
}
| { type: 'resumed' }
| { type: 'ended'; endReason: ScheduleEndReason };
/** A schedule that keeps running, one that runs once, or a delayed notification. */
export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
/**
* A schedule as its holder sees it. Times are epoch ms. `context` values are
* never returned, as for a persistent subscription: only its keys and a hash.
*/
export interface ScheduleView {
scheduleId: string;
kind: ScheduleViewKind;
appUid: string;
/** `null` once ended: the handler binding goes with the schedule. */
handlerName: string | null;
/** Set only when the handler lives in the holder's own scope. */
handlerScope?: 'user';
contextKeys: string[] | null;
contextHash: string | null;
/** Interval of an `every` schedule. */
every: number | null;
cron: string | null;
tz: string;
/** First allowed run; a one-shot's only run. */
startAt: number;
/** Last allowed run, inclusive. */
endsAt: number | null;
/** `sending` once a delayed send has started; it can't be cancelled then. */
state: 'active' | 'suspended' | 'sending' | 'ended';
suspendedReason: string | null;
endReason: string | null;
endedAt: number | null;
/**
* The slot it runs for next; `null` while suspended or sending, and once
* ended.
*/
nextRunAt: number | null;
createdAt: number;
}
export interface ScheduleListRequest {
limit: number;
cursor?: string;
includeTotal: boolean;
}
/** The caps a holder's schedules are held to, from their plan. */
export interface ScheduleQuota {
recurringPerUser: number;
recurringPerApp: number;
oneShotsPerUser: number;
oneShotsPerApp: number;
/** Worst-case runs per hour × 1000, summed over recurring rows. */
budgetPerUser: number;
budgetPerApp: number;
}
/** A caller cleared to supply `sendAt`, and when it asked for. */
export interface DeferredSendPlan {
sendAt: number;
/** Due within 5 seconds, or past: sent now rather than stored. */
immediate: boolean;
holderUserId: number;
appUid: string;
handlerDepth: number;
quota: ScheduleQuota;
}
/** What a delayed app message stores until it is sent. */
export interface DeferredAppMessage {
message: AppMessage;
/** A KV share handle naming the recipient. */
to?: string;
}
/**
* A condition on one row of another table that statements embed as `EXISTS
* (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
*/
export interface SqlGuard {
table: string;
where: string;
values: unknown[];
}
/** Ceiling of a schedule listing page. */
export const SCHEDULE_LIST_LIMIT_CAP = 200;
/** The signed peer route a region's schedule changes are sent to. */
export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
export const schedulesUnavailable = (): HttpError =>
new HttpError(503, 'Schedules are not available on this server', {
legacyCode: 'events_disabled',
// The designed answer of a server without schedules, not a fault.
noAlarm: true,
});
export interface ScheduleServiceContract {
/**
* Keep schedules in step with a change to the subscription rows they own.
* Never throws: the subscription change has already happened.
*/
onSubscriptionsChanged(
rows: readonly DurableSubscription[],
change: ScheduleSubscriptionChange,
): Promise<void>;
createFor(
actor: Actor,
body: Record<string, unknown>,
): Promise<ScheduleView>;
list(
actor: Actor,
request: ScheduleListRequest,
): Promise<PageResult<ScheduleView>>;
get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
update(
actor: Actor,
scheduleId: unknown,
changes: unknown,
): Promise<ScheduleView>;
remove(actor: Actor, scheduleId: unknown): Promise<void>;
/** `results`: each change's index outcome, in order (`null` if dropped). */
receivePeerChanges(
body: unknown,
): Promise<{ applied: number; results?: Array<number | null> }>;
/** Checks a caller supplying `sendAt`; `events_disabled` when off. */
prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
createDeferredAppMessage(
plan: DeferredSendPlan,
message: DeferredAppMessage,
): Promise<{ scheduleId: string; sendAt: number }>;
/**
* Runs an events worker deploy under the schedules' deploy gate; `capped`
* for a schedule run's. A passthrough when schedules are off.
*/
gateDeploy?(
script: string,
options: { capped: boolean },
deploy: () => Promise<EventsDeployOutcome>,
): Promise<EventsDeployOutcome>;
}
@@ -209,7 +209,6 @@ afterAll(async () => {
const attemptsFor = async (
model: string,
thrown: unknown = new Error('upstream down'),
extra: { provider?: string; fallback?: boolean } = {},
) => {
createMock.mockRejectedValue(thrown);
let caught: HttpError | undefined;
@@ -218,7 +217,6 @@ const attemptsFor = async (
driver.complete({
model,
messages: [{ role: 'user', content: 'hi' }],
...extra,
}),
);
} catch (e) {
@@ -888,59 +886,6 @@ describe('ChatCompletionDriver duplicate-model fallback', () => {
});
});
describe('ChatCompletionDriver fallback opt-out', () => {
const SHARED = 'deepseek-v4-pro';
it('fails on the first route when the caller sets fallback: false', async () => {
const attempts = await attemptsFor(SHARED, undefined, {
fallback: false,
});
expect(attempts).toHaveLength(1);
expect(attempts[0]).toMatchObject({ provider: 'deepseek' });
});
it('tries only the pinned provider', async () => {
const attempts = await attemptsFor(SHARED, undefined, {
provider: 'openrouter',
fallback: false,
});
expect(attempts).toHaveLength(1);
expect(attempts[0]).toMatchObject({ provider: 'openrouter' });
});
it('rejects a pinned provider that has no route for the model', async () => {
createMock.mockClear();
await expect(
withTestActor(() =>
driver.complete({
model: SHARED,
provider: 'claude',
fallback: false,
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
it('still marks the failed route for the next caller', async () => {
const [only] = await attemptsFor(SHARED, undefined, {
fallback: false,
});
expect(isRouteUnhealthy(only.provider, only.model)).toBe(true);
});
it('keeps the chain when fallback is true or unset', async () => {
expect(
await attemptsFor(SHARED, undefined, { fallback: true }),
).toHaveLength(3);
expect(await attemptsFor(SHARED)).toHaveLength(3);
});
});
// Each attempt in a fallback chain is a whole completion at that model's
// prices, so each one has to clear the credit gate on its own. The chain used
// to re-check with a nominal 1-microcent amount, which any account with a
@@ -709,20 +709,6 @@ export class ChatCompletionDriver extends PuterDriver {
legacyCode: 'bad_request',
});
}
// A pinned provider without a route for the model normally resolves
// to another vendor; with `fallback: false` that is the one thing the
// caller asked us not to do.
if (
args.fallback === false &&
intendedProvider &&
model.provider !== intendedProvider
) {
throw new HttpError(
400,
`Model ${args.model} is not served by provider ${intendedProvider}`,
{ legacyCode: 'bad_request' },
);
}
args.messages = normalizeMediaParts(normalize_messages(args.messages));
if (args.tools) {
@@ -960,15 +946,8 @@ export class ChatCompletionDriver extends PuterDriver {
const bucketKey = model.id;
const tried = new Set([routeId(model.provider!, model.id)]);
let lastError: Error | null = e as Error;
// With `fallback: false` the caller wants this route's answer or
// its error, not a bill for another provider's attempt.
const fallbackAllowed = args.fallback !== false;
while (
fallbackAllowed &&
lastError &&
attempts.length < MAX_ATTEMPTS
) {
while (lastError && attempts.length < MAX_ATTEMPTS) {
const fallback = this.#findFallback(bucketKey, tried);
if (!fallback) break;
@@ -809,11 +809,6 @@ describe('Alibaba official catalog', () => {
expect(models.some((model) => model.id === 'qwen-image-fixture')).toBe(
false,
);
const fixture = models.find(
(model) => model.id === 'qwen-catalog-fixture',
)!;
expect(fixture.puterId).toBe('alibaba:qwen/qwen-catalog-fixture');
expect(modelLookupNames(models)).toContain('qwen-catalog-fixture');
const plus = models.find((model) => model.id === 'qwen-plus')!;
const original = ALIBABA_MODELS.find(
(model) => model.id === 'qwen-plus',
@@ -914,115 +909,6 @@ describe('Alibaba official catalog', () => {
expect(model.costs).toEqual({ tokens: 1_000_000 });
});
it.each([
['qwen3.9-plus-2027-01-01', 'alibaba:qwen/qwen3.9-plus-2027-01-01'],
['kimi/kimi-k3', 'alibaba:kimi/kimi-k3'],
[
'deepseek-v4.1-flash-1017',
'alibaba:deepseek/deepseek-v4.1-flash-1017',
],
['glm-5.3-prime', 'alibaba:zhipu/glm-5.3-prime'],
['decision-model-preview', 'alibaba:qwen/decision-model-preview'],
])(
'gives uncurated model %s a puterId in the alibaba:<vendor>/<id> form',
(id, puterId) => {
const model = mapAlibabaCatalogModel({
model: id,
capabilities: ['TG'],
})!;
expect(model.puterId).toBe(puterId);
},
);
it('skips a vendor-prefixed spelling of a curated model', () => {
const model = mapAlibabaCatalogModel({
model: 'zhipu/glm-5.2',
capabilities: ['TG', 'Reasoning'],
inference_metadata: {
request_modality: ['Text'],
response_modality: ['Text'],
},
});
expect(model).toBeNull();
});
it.each([
[
'ASR',
{
capabilities: ['ASR'],
inference_metadata: {
request_modality: ['Audio'],
response_modality: ['Text'],
},
},
],
[
'rerank',
{
capabilities: ['TR'],
inference_metadata: {
request_modality: ['Text'],
response_modality: ['Text'],
},
},
],
[
'image generation',
{
capabilities: ['IG'],
inference_metadata: {
request_modality: ['Text'],
response_modality: ['Text', 'Image'],
},
},
],
[
'realtime omni',
{
capabilities: ['Realtime-Omni'],
inference_metadata: {
request_modality: ['Text', 'Image', 'Audio', 'Video'],
response_modality: ['Text', 'Audio'],
},
},
],
[
'TTS',
{
capabilities: ['TTS'],
inference_metadata: {
request_modality: ['Audio'],
response_modality: ['Text'],
},
},
],
])('rejects %s models that cannot serve chat completions', (_kind, raw) => {
expect(
mapAlibabaCatalogModel({
model: 'non-chat-fixture',
...(raw as AlibabaApiModel),
}),
).toBeNull();
});
it('admits a vision model', () => {
const model = mapAlibabaCatalogModel({
model: 'vision-fixture',
capabilities: ['VU'],
inference_metadata: {
request_modality: ['Text', 'Image'],
response_modality: ['Text'],
},
})!;
expect(model.modalities?.input).toEqual(['text', 'image']);
});
it('does not admit an uncurated entry without capability or modality data', () => {
expect(mapAlibabaCatalogModel({ model: 'mystery-model' })).toBeNull();
expect(mapAlibabaCatalogModel({ model: 'qwen-plus' })).not.toBeNull();
});
it('refuses inference with unverified catalog prices', async () => {
seedPages();
const { provider } = makeProvider();
@@ -82,23 +82,6 @@ const positiveLimit = (value: unknown): number | undefined =>
? value
: undefined;
const chatCapabilities = ['TG', 'Reasoning', 'VU'];
const curatedPuterIds = new Set(ALIBABA_MODELS.map((model) => model.puterId));
const inferPuterId = (id: string): string => {
if (id.includes('/')) return `alibaba:${id}`;
const family = id.split('-')[0];
const sibling = ALIBABA_MODELS.find(
(model) => model.puterId && model.id.split('-')[0] === family,
);
// Unprefixed feed ids are Model Studio's own family unless a curated
// sibling points at a third-party org.
const org =
sibling?.puterId?.slice('alibaba:'.length).split('/')[0] ?? 'qwen';
return `alibaba:${org}/${id}`;
};
export const mapAlibabaCatalogModel = (
raw: AlibabaApiModel,
previous?: IChatModel,
@@ -107,25 +90,16 @@ export const mapAlibabaCatalogModel = (
const id = raw.model.trim();
const existing =
previous ?? ALIBABA_MODELS.find((model) => model.id === id);
const input = raw.inference_metadata?.request_modality;
const output = raw.inference_metadata?.response_modality;
if (output && !output.includes('Text')) return null;
if (
raw.capabilities &&
!raw.capabilities.some((capability) =>
chatCapabilities.includes(capability),
!output &&
!existing &&
!raw.capabilities?.some((capability) =>
['TG', 'Reasoning', 'VU'].includes(capability),
)
)
return null;
if (input && !input.includes('Text')) return null;
if (output && !output.includes('Text')) return null;
if (!raw.capabilities && !input && !output && !existing) return null;
let puterId = existing?.puterId;
if (!puterId) {
puterId = inferPuterId(id);
// A feed entry spelling a curated model differently would duplicate
// the curated entry's puterId and list the model twice.
if (curatedPuterIds.has(puterId)) return null;
}
const catalogPricing: AlibabaCatalogTier[] = (raw.prices ?? []).map(
(tier) => {
// Abbreviated ranges do not specify whether k means 1000 or 1024 tokens.
@@ -175,7 +149,6 @@ export const mapAlibabaCatalogModel = (
return {
...existing,
id,
puterId,
name: raw.name ?? existing?.name ?? id,
description: raw.description ?? existing?.description,
aliases: existing?.aliases,
-5
View File
@@ -175,11 +175,6 @@ export interface UsageDetails {
export interface ICompleteArguments {
messages: PuterMessage[];
provider?: string;
/**
* `false` makes the first route's failure final instead of retrying the
* request on another provider that serves the same model.
*/
fallback?: boolean;
stream?: boolean;
model: string;
test_mode?: boolean;
@@ -39,49 +39,9 @@ describe('estimateTextTokens', () => {
const long = estimateTextTokens('hello world '.repeat(100));
expect(long).toBeGreaterThan(short);
});
it('counts words the way splitting on \\s+ does', () => {
const splitEstimate = (text: string) =>
Math.floor(
(text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2,
);
for (const text of [
'a',
' a',
'a ',
' a b\t\nc ',
'x y z w 

   v',
'é ü 日本語 テキスト',
'\r\n\v\f',
]) {
expect(estimateTextTokens(text), JSON.stringify(text)).toBe(
splitEstimate(text),
);
}
});
it('gives text past the scan budget the word density scanned so far', () => {
const text = 'ab cd '.repeat(1000);
const budget = { chars: 12, scannedChars: 0, scannedRuns: 0 };
// Uniform text extrapolates to the full count.
expect(estimateTextTokens(text, budget)).toBe(estimateTextTokens(text));
expect(budget.chars).toBe(0);
// A later part with no budget left takes the same density.
expect(estimateTextTokens(text, budget)).toBe(estimateTextTokens(text));
});
});
describe('estimatePromptTokens', () => {
it('scans a bounded amount of text per prompt', () => {
// Many parts, each under the budget alone, past it together; the
// unscanned ones are extrapolated, which is exact for uniform text.
const part = 'lorem ipsum dolor '.repeat(30_000);
const parts = Array.from({ length: 8 }, () => ({ text: part }));
expect(estimatePromptTokens([{ role: 'user', content: parts }])).toBe(
8 * estimateTextTokens(part),
);
});
it('counts plain string content', () => {
const tokens = estimatePromptTokens([
{ role: 'user', content: 'a fairly ordinary sentence to price' },
@@ -29,11 +29,7 @@
* passes every gate.
*/
import {
createWordScanBudget,
estimateTextTokens,
type WordScanBudget,
} from '../../util/tokenEstimate.js';
import { estimateTextTokens } from '../../util/tokenEstimate.js';
// Re-exported so existing consumers keep one import site for all estimators.
export { estimateTextTokens } from '../../util/tokenEstimate.js';
@@ -95,25 +91,23 @@ const attachmentTokens = (value: unknown): number => {
* recognises is charged the unmeasured estimate — the alternative is a content
* type nobody has taught the gate about arriving for free.
*/
const partTokens = (part: unknown, budget: WordScanBudget): number => {
if (typeof part === 'string') return estimateTextTokens(part, budget);
const partTokens = (part: unknown): number => {
if (typeof part === 'string') return estimateTextTokens(part);
if (!part || typeof part !== 'object') return 0;
const p = part as ContentPart;
if (typeof p.text === 'string') return estimateTextTokens(p.text, budget);
if (typeof p.text === 'string') return estimateTextTokens(p.text);
// Model-output shapes replayed as history: Anthropic thinking blocks,
// our own streamed `reasoning` chunks, OpenAI refusals. All text — they
// must not fall through to the attachment default below, which would
// price a few sentences of reasoning like a full-frame image.
if (typeof p.thinking === 'string')
return estimateTextTokens(p.thinking, budget);
if (typeof p.thinking === 'string') return estimateTextTokens(p.thinking);
if (typeof p.reasoning === 'string') {
return estimateTextTokens(p.reasoning, budget);
return estimateTextTokens(p.reasoning);
}
if (typeof p.refusal === 'string')
return estimateTextTokens(p.refusal, budget);
if (typeof p.refusal === 'string') return estimateTextTokens(p.refusal);
// OpenAI-style `{ image_url: { url } }` (or the flattened string form).
if (p.image_url) {
@@ -142,7 +136,6 @@ const partTokens = (part: unknown, budget: WordScanBudget): number => {
if (p.type === 'tool_use' && p.input !== undefined) {
return estimateTextTokens(
typeof p.input === 'string' ? p.input : JSON.stringify(p.input),
budget,
);
}
if (p.type === 'tool_result') {
@@ -150,11 +143,10 @@ const partTokens = (part: unknown, budget: WordScanBudget): number => {
typeof p.content === 'string'
? p.content
: JSON.stringify(p.content ?? ''),
budget,
);
}
if (p.type === 'compaction') {
return estimateTextTokens(String(p.encrypted_content ?? ''), budget);
return estimateTextTokens(String(p.encrypted_content ?? ''));
}
return UNMEASURED_ATTACHMENT_TOKENS;
@@ -172,22 +164,21 @@ const partTokens = (part: unknown, budget: WordScanBudget): number => {
export const estimatePromptTokens = (messages: unknown): number => {
if (!Array.isArray(messages)) return 0;
const budget = createWordScanBudget();
let tokens = 0;
for (const message of messages) {
if (typeof message === 'string') {
tokens += estimateTextTokens(message, budget);
tokens += estimateTextTokens(message);
continue;
}
if (!message || typeof message !== 'object') continue;
const content = (message as ContentPart).content;
if (typeof content === 'string') {
tokens += estimateTextTokens(content, budget);
tokens += estimateTextTokens(content);
} else if (Array.isArray(content)) {
for (const part of content) tokens += partTokens(part, budget);
for (const part of content) tokens += partTokens(part);
} else if (content) {
tokens += partTokens(content, budget);
tokens += partTokens(content);
}
}
return tokens;
@@ -463,26 +463,10 @@ describe('ImageGenerationDriver.generate provider routing', () => {
driver.generate({ prompt: 'a landscape' }),
);
expect(openaiImagesGenerateMock).toHaveBeenCalledWith(
expect.objectContaining({ model: 'gpt-image-2.6-flare' }),
expect.objectContaining({ model: 'gpt-image-2' }),
);
});
it.each(['gpt-image-2.6-flare', 'gpt-image-2.6-sunburst'])(
'routes %s and its OpenAI alias', async (model) => {
for (const id of [model, `openai/${model}`]) {
openaiImagesGenerateMock.mockResolvedValueOnce({
data: [{ url: 'https://oai/image.png' }],
});
await withDriverName('ai-image', () => driver.generate({
model: id, prompt: 'a landscape',
}));
expect(openaiImagesGenerateMock).toHaveBeenLastCalledWith(
expect.objectContaining({ model }),
);
}
},
);
it('rejects an excluded model regardless of caller casing', async () => {
await expect(withActor(() =>
driver.generate({
@@ -150,9 +150,9 @@ describe('OpenAiImageProvider construction', () => {
// ── Model catalog ───────────────────────────────────────────────────
describe('OpenAiImageProvider model catalog', () => {
it('returns gpt-image-2.6-flare as the default', () => {
it('returns gpt-image-2 as the default', () => {
const provider = makeProvider();
expect(provider.getDefaultModel()).toBe('gpt-image-2.6-flare');
expect(provider.getDefaultModel()).toBe('gpt-image-2');
});
it('drops the deprecated gpt-image-1 family and chatgpt-image-latest', () => {
@@ -235,56 +235,6 @@ describe('OpenAiImageProvider.generate argument validation', () => {
// ── Credit gate ─────────────────────────────────────────────────────
describe('OpenAiImageProvider.generate credit gate', () => {
it('waits for the real usage write before returning an image', async () => {
const actor = makeActor({
user: {
id: 62,
uuid: `image-write-${Date.now()}`,
username: 'metered',
},
});
const redis = server.clients.redis as unknown as {
meterIncr: (...args: string[]) => Promise<unknown>;
};
const realIncr = redis.meterIncr.bind(redis);
let release!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
let blocked = false;
vi.spyOn(redis, 'meterIncr').mockImplementation(async (...args) => {
if (!blocked && args[5].includes(`:actor:${actor.user!.uuid}:`)) {
blocked = true;
await held;
}
return realIncr(...args);
});
generateMock.mockResolvedValueOnce({
data: [{ url: 'https://image.example/result.png' }],
});
const provider = makeProvider();
let finished = false;
const generating = withTestActor(
() =>
provider.generate({
model: 'gpt-image-2',
prompt: 'an image',
ratio: { w: 1024, h: 1024 },
}),
actor,
).then((result) => {
finished = true;
return result;
});
try {
await vi.waitFor(() => expect(blocked).toBe(true));
expect(finished).toBe(false);
} finally {
release();
await generating;
}
});
it('throws 402 BEFORE hitting OpenAI when actor lacks credits', async () => {
const provider = makeProvider();
hasCreditsSpy.mockResolvedValueOnce(false);
@@ -907,64 +857,3 @@ it('keeps a landscape aspect hint landscape at gpt-image-2 sizes', async () => {
expect(w).toBeGreaterThan(h);
expect(Math.abs(w / h - 1.7)).toBeLessThan(0.1);
});
describe('OpenAiImageProvider GPT Image 2.6', () => {
it.each(['gpt-image-2.6-flare', 'gpt-image-2.6-sunburst'])(
'%s generates and edits with token-based billing', async (model) => {
const provider = makeProvider();
const response = {
data: [{ b64_json: 'AAAA' }],
usage: {
input_tokens: 100,
output_tokens: 800,
input_tokens_details: { text_tokens: 100, image_tokens: 0 },
},
};
generateMock.mockResolvedValue(response);
editMock.mockResolvedValue(response);
for (const input_image of [undefined, 'data:image/png;base64,AAAA']) {
const result = await withTestActor(() => provider.generate({
model, prompt: 'hi', quality: 'xhigh',
ratio: { w: 1024, h: 1024 }, input_image,
}));
expect(result).toBe('data:image/png;base64,AAAA');
}
for (const mock of [generateMock, editMock]) {
expect(mock.mock.calls[0]![0]).toMatchObject({
model, quality: 'xhigh', size: '1024x1024',
});
}
const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
expect(entries).toEqual(expect.arrayContaining([
expect.objectContaining({
usageType: `openai:${model}:xhigh:1024x1024:output`,
costOverride: 2_400_000,
}),
]));
},
);
it.each([
['low', 1024, 1024, 823],
['medium', 1024, 1024, 1449],
['high', 1024, 1024, 1756],
['xhigh', 1024, 1024, 3122],
['max', 1024, 1024, 7024],
['auto', 1024, 1024, 1449],
['low', 2048, 1088, 571],
])('estimates %s at %sx%s from the 2.6 calculator', async (quality, w, h, tokens) => {
generateMock.mockResolvedValue({ data: [{ b64_json: 'AAAA' }] });
await withTestActor(() => makeProvider().generate({
model: 'gpt-image-2.6-flare', prompt: 'hi', quality: String(quality),
ratio: { w: Number(w), h: Number(h) },
}));
const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
const output = (entries as Array<{ usageType: string; costOverride: number }>)
.find((entry) => entry.usageType.endsWith(':output'))!;
expect(output.usageType).toBe(`openai:gpt-image-2.6-flare:${quality}:${w}x${h}:output`);
// The metering conversion rounds fractional microcents upward.
expect(output.costOverride).toBeGreaterThanOrEqual(Number(tokens) * 3000);
expect(output.costOverride).toBeLessThanOrEqual(Number(tokens) * 3000 + 1);
});
});
@@ -56,8 +56,8 @@ interface OpenAIImageUsage {
/**
* OpenAI image generation provider for v2. Supports the GPT Image models
* (gpt-image-2 and the 2.5/2.6 Sunburst and Flare models), including
* image-to-image editing via `input_images` (the `images.edit` endpoint).
* (gpt-image-2, -2.5-sunburst, -2.5-flare), including image-to-image editing
* via `input_images` (the `images.edit` endpoint).
*/
export class OpenAiImageProvider implements IImageProvider {
#meteringService: AiMeteringService;
@@ -109,7 +109,7 @@ export class OpenAiImageProvider implements IImageProvider {
}
getDefaultModel(): string {
return 'gpt-image-2.6-flare';
return 'gpt-image-2';
}
async generate({
@@ -293,10 +293,7 @@ export class OpenAiImageProvider implements IImageProvider {
});
}
if (usageEntries.length) {
await this.#meteringService.batchIncrementUsages(
actor,
usageEntries,
);
this.#meteringService.batchIncrementUsages(actor, usageEntries);
}
const url =
@@ -604,36 +601,17 @@ export class OpenAiImageProvider implements IImageProvider {
height: number,
quality?: string,
): number {
const isGptImage26 = model.startsWith('gpt-image-2.6');
const factors =
isGptImage26 || model.startsWith('gpt-image-2.5')
? OpenAiImageProvider.#GPT_IMAGE_2_5_LATENT_FACTORS
: OpenAiImageProvider.#GPT_IMAGE_2_LATENT_FACTORS;
const factors = model.startsWith('gpt-image-2.5')
? OpenAiImageProvider.#GPT_IMAGE_2_5_LATENT_FACTORS
: OpenAiImageProvider.#GPT_IMAGE_2_LATENT_FACTORS;
const factor = factors[quality ?? ''] ?? factors.medium;
const longEdge = Math.max(width, height);
const shortEdge = Math.min(width, height);
const shortGrid = (factor * shortEdge) / longEdge;
const floorGrid = Math.floor(shortGrid);
const shortLatent =
isGptImage26 && shortGrid - floorGrid === 0.5
? floorGrid + (floorGrid % 2)
: Math.round(shortGrid);
const shortLatent = Math.round((factor * shortEdge) / longEdge);
const latentW = width >= height ? factor : shortLatent;
const latentH = width >= height ? shortLatent : factor;
const baseArea = latentW * latentH;
const tokens = Math.ceil(
(baseArea * (2_000_000 + width * height)) / 4_000_000,
);
// GPT Image 2.6 bills more tokens at low and medium quality.
const estimatedQuality =
quality && quality in factors ? quality : 'medium';
const multiplier =
estimatedQuality === 'low'
? 4.2
: estimatedQuality === 'medium'
? 3.3
: 1;
return isGptImage26 ? Math.round(tokens * multiplier) : tokens;
return Math.ceil((baseArea * (2_000_000 + width * height)) / 4_000_000);
}
#estimateOutputCostFromTokens(
@@ -20,46 +20,6 @@
import type { IImageModel } from '../../types.js';
export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [
{
puterId: 'openai:openai/gpt-image-2.6-sunburst',
id: 'gpt-image-2.6-sunburst',
aliases: ['openai/gpt-image-2.6-sunburst'],
name: 'GPT Image 2.6 Sunburst',
version: '2.6',
costs_currency: 'usd-cents',
index_cost_key: 'low:1024x1024',
costs: {
// Text tokens (per 1M tokens)
text_input: 500, // $5.00
text_cached_input: 125, // $1.25
// Image tokens (per 1M tokens)
image_input: 800, // $8.00
image_cached_input: 200, // $2.00
image_output: 3000, // $30.00
'low:1024x1024': 2.469,
},
allowedQualityLevels: ['low', 'medium', 'high', 'xhigh', 'max', 'auto'],
},
{
puterId: 'openai:openai/gpt-image-2.6-flare',
id: 'gpt-image-2.6-flare',
aliases: ['openai/gpt-image-2.6-flare'],
name: 'GPT Image 2.6 Flare',
version: '2.6',
costs_currency: 'usd-cents',
index_cost_key: 'low:1024x1024',
costs: {
// Text tokens (per 1M tokens)
text_input: 500, // $5.00
text_cached_input: 125, // $1.25
// Image tokens (per 1M tokens)
image_input: 800, // $8.00
image_cached_input: 200, // $2.00
image_output: 3000, // $30.00
'low:1024x1024': 2.469,
},
allowedQualityLevels: ['low', 'medium', 'high', 'xhigh', 'max', 'auto'],
},
{
puterId: 'openai:openai/gpt-image-2.5-sunburst',
id: 'gpt-image-2.5-sunburst',
+54 -107
View File
@@ -31,13 +31,11 @@ import {
normalizeRawBase64ImageString,
validateIconDataUrl,
} from '../../util/appIcon.js';
import { isOriginBootstrapApp, toAppReadView } from '../../util/appView.js';
import { awaitLater } from '../../util/concurrency.js';
import { toAppReadView } from '../../util/appView.js';
import { isUniqueViolation } from '../../util/dbError.js';
import {
buildHostedSubdomainIndexUrlCandidates,
extractPuterHostedSubdomain,
resolveCustomHostSubdomain,
} from '../../util/hostedAppBacking.js';
import {
normalizeLimit,
@@ -203,38 +201,30 @@ export class AppDriver extends PuterDriver {
const fields = await this.#validateInput(object, { isCreate: true });
// Puter-hosted index_url handling (custom domains included). Order
// matches v1 AppES:
// 1. Refuse if the index_url's site isn't owned by this user.
// Puter-hosted index_url handling. Order matches v1 AppES:
// 1. Refuse if the index_url's subdomain isn't owned by this user.
// 2. Try to merge into an existing row with the same index_url
// (origin-bootstrap takeover, or claiming an unowned row).
// 3. Otherwise enforce index_url uniqueness so two rows can't
// share a hosted URL.
// The name lookup runs alongside so the host checks add no round
// trip of their own.
const nameTaken = awaitLater(this.appStore.existsByName(fields.name));
const customSite = await this.#customSiteForIndexUrl(fields.index_url);
await this.#ensurePuterSiteSubdomainIsOwned(
fields.index_url,
actor.user,
customSite,
);
const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlApp({
object,
options,
user: actor.user,
customSite,
});
if (joinedApp) {
return joinedApp;
}
await this.#ensureIndexUrlNotAlreadyInUse({
indexUrl: fields.index_url,
customSite,
});
// Name conflict handling
if (await nameTaken) {
if (await this.appStore.existsByName(fields.name)) {
if (options?.dedupe_name) {
let candidate;
let i = 0;
@@ -434,23 +424,13 @@ export class AppDriver extends PuterDriver {
existing: app,
});
// Started before the index_url checks so they add no round trip.
const renaming = !!fields.name && fields.name !== app.name;
const nameTaken = renaming
? awaitLater(this.appStore.existsByName(fields.name))
: null;
// Puter-hosted index_url handling on update — same flow as create.
// #validateInput already dropped an unchanged index_url. Self-app is
// excluded from the conflict search via `excludeAppId`.
if (fields.index_url) {
const customSite = await this.#customSiteForIndexUrl(
fields.index_url,
);
await this.#ensurePuterSiteSubdomainIsOwned(
fields.index_url,
actor.user,
customSite,
);
const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlApp({
object,
@@ -458,7 +438,6 @@ export class AppDriver extends PuterDriver {
user: actor.user,
sourceAppUid: app.uid,
excludeAppId: app.id,
customSite,
});
if (joinedApp) {
return joinedApp;
@@ -466,7 +445,6 @@ export class AppDriver extends PuterDriver {
await this.#ensureIndexUrlNotAlreadyInUse({
indexUrl: fields.index_url,
excludeAppId: app.id,
customSite,
});
if (
app.is_private &&
@@ -487,10 +465,14 @@ export class AppDriver extends PuterDriver {
}
// Name conflict check (only if name is changing)
if (nameTaken && (await nameTaken)) {
throw new HttpError(409, 'An app with this name already exists', {
legacyCode: 'conflict',
});
if (fields.name && fields.name !== app.name) {
if (await this.appStore.existsByName(fields.name)) {
throw new HttpError(
409,
'An app with this name already exists',
{ legacyCode: 'conflict' },
);
}
}
const filetypes = fields.filetype_associations;
@@ -881,26 +863,6 @@ export class AppDriver extends PuterDriver {
return !!this.#extractPuterHostedSubdomain(indexUrl);
}
/**
* `{ host, subdomain }` when `indexUrl` is on a custom domain that serves
* one of our sites, else null. Such a host gets the hosted-subdomain rules:
* owner-only, one app per URL, bootstrap rows merged.
*/
async #customSiteForIndexUrl(indexUrl) {
if (!indexUrl || this.#isPuterHostedIndexUrl(indexUrl)) return null;
const parsed = URL.parse(indexUrl);
if (parsed?.protocol !== 'https:' && parsed?.protocol !== 'http:') {
return null;
}
const host = parsed.hostname.toLowerCase().replace(/\.+$/, '');
const subdomain = await resolveCustomHostSubdomain(
host,
this.config,
this.clients.event,
);
return subdomain ? { host, subdomain } : null;
}
/**
* Read normalized origin-alias groups from config. Each group is a deduped
* list of lowercased, trimmed bare hosts. Malformed entries are skipped so
@@ -980,18 +942,9 @@ export class AppDriver extends PuterDriver {
}
}
async #findIndexUrlConflictRow({
indexUrl,
excludeAppId,
customSite,
} = {}) {
async #findIndexUrlConflictRow({ indexUrl, excludeAppId } = {}) {
const aliasGroup = this.#findOriginAliasGroupForIndexUrl(indexUrl);
if (
!this.#isPuterHostedIndexUrl(indexUrl) &&
!aliasGroup &&
!customSite
)
return null;
if (!this.#isPuterHostedIndexUrl(indexUrl) && !aliasGroup) return null;
const candidates = new Set(
this.#buildEquivalentIndexUrlCandidates(indexUrl),
@@ -1010,18 +963,15 @@ export class AppDriver extends PuterDriver {
// For alias-group hosts, treat the group as a host-level reservation:
// any row whose index_url is the root URL of any group member counts
// as a conflict, so a single app owns the whole group. A custom
// domain's root is reserved the same way.
const reservedHosts = [
...(aliasGroup ?? []),
...(customSite ? [customSite.host] : []),
];
for (const host of reservedHosts) {
for (const proto of ['https', 'http']) {
const base = `${proto}://${host}`;
candidates.add(base);
candidates.add(`${base}/`);
candidates.add(`${base}/index.html`);
// as a conflict, so a single app owns the whole group.
if (aliasGroup) {
for (const host of aliasGroup) {
for (const proto of ['https', 'http']) {
const base = `${proto}://${host}`;
candidates.add(base);
candidates.add(`${base}/`);
candidates.add(`${base}/index.html`);
}
}
}
@@ -1034,15 +984,10 @@ export class AppDriver extends PuterDriver {
});
}
async #ensureIndexUrlNotAlreadyInUse({
indexUrl,
excludeAppId,
customSite,
} = {}) {
async #ensureIndexUrlNotAlreadyInUse({ indexUrl, excludeAppId } = {}) {
const conflictRow = await this.#findIndexUrlConflictRow({
indexUrl,
excludeAppId,
customSite,
});
if (conflictRow) {
throw new HttpError(400, 'App index_url already in use', {
@@ -1055,11 +1000,9 @@ export class AppDriver extends PuterDriver {
}
}
async #ensurePuterSiteSubdomainIsOwned(indexUrl, user, customSite) {
async #ensurePuterSiteSubdomainIsOwned(indexUrl, user) {
if (!user) return;
const subdomain =
customSite?.subdomain ??
this.#extractPuterHostedSubdomain(indexUrl);
const subdomain = this.#extractPuterHostedSubdomain(indexUrl);
if (!subdomain) return;
let row = await this.stores.subdomain.getBySubdomain(subdomain);
@@ -1072,22 +1015,29 @@ export class AppDriver extends PuterDriver {
});
}
if (!row || row.user_id !== user.id) {
// A custom domain's site name isn't the caller's to learn.
throw new HttpError(
400,
customSite
? 'Domain not owned by user'
: 'Subdomain not owned by user',
{
legacyCode: 'subdomain_not_owned',
fields: customSite
? { hostname: customSite.host }
: { subdomain },
},
);
throw new HttpError(400, 'Subdomain not owned by user', {
legacyCode: 'subdomain_not_owned',
fields: { subdomain },
});
}
}
/**
* Origin-bootstrap detection: rows auto-created when an unknown origin
* first needed an app row (no human-supplied metadata). Marker is `name ===
* uid && title === uid` and a description starting with "App created from
* origin ". Only these rows are eligible for same-owner merging — refusing
* to merge arbitrary same-owner apps prevents accidental data loss.
*/
#isOriginBootstrapApp(app) {
if (!app || typeof app !== 'object') return false;
if (typeof app.uid !== 'string' || !app.uid) return false;
if (app.name !== app.uid) return false;
if (app.title !== app.uid) return false;
if (typeof app.description !== 'string') return false;
return app.description.startsWith('App created from origin ');
}
// -- Canonical-uid alias kvstore pair -----------------------------
//
// After a merge, the source app uid → canonical uid mapping is
@@ -1181,19 +1131,17 @@ export class AppDriver extends PuterDriver {
user,
sourceAppUid,
excludeAppId,
customSite,
} = {}) {
const indexUrl = object?.index_url;
// Alias-group hosts (`app_origin_aliases`) and custom domains get the
// same merge treatment as puter-hosted subdomains. Without this, a
// bootstrap stub on such a host could never be absorbed — and since
// `#findIndexUrlConflictRow` *does* honor them, the uniqueness check
// below would hard-reject the owner's own create/update instead of
// merging into the stub.
// Alias-group hosts (`app_origin_aliases`) get the same merge
// treatment as puter-hosted subdomains. Without this, a bootstrap
// stub on a custom domain could never be absorbed — and since
// `#findIndexUrlConflictRow` *does* honor alias groups, the
// uniqueness check below would hard-reject the owner's own
// create/update instead of merging into the stub.
if (
!this.#isPuterHostedIndexUrl(indexUrl) &&
!this.#findOriginAliasGroupForIndexUrl(indexUrl) &&
!customSite
!this.#findOriginAliasGroupForIndexUrl(indexUrl)
) {
return null;
}
@@ -1201,7 +1149,6 @@ export class AppDriver extends PuterDriver {
const conflictRow = await this.#findIndexUrlConflictRow({
indexUrl,
excludeAppId,
customSite,
});
if (!conflictRow) return null;
@@ -1250,7 +1197,7 @@ export class AppDriver extends PuterDriver {
if (
Number.isInteger(conflictOwnerUserId) &&
conflictOwnerUserId === user.id &&
!isOriginBootstrapApp(appToJoin)
!this.#isOriginBootstrapApp(appToJoin)
) {
// Prevent merging arbitrary same-owner apps; only allow the
// auto-created origin bootstrap row to be absorbed.
-182
View File
@@ -1912,188 +1912,6 @@ describe('AppDriver alias-group index_url merge', () => {
});
});
// ── custom-domain index_url rules ───────────────────────────────────
//
// A host a `site.resolveHost` listener maps to a site gets the hosted
// rules: only the site owner may point an app at it, one app per URL, and
// creating there absorbs the origin-bootstrap row.
describe('AppDriver custom-domain index_url', () => {
const hosts = new Map<string, string>();
let failResolve = false;
const resolveHost = (
_key: unknown,
data: { host: string; result: { subdomain: string | null } },
) => {
if (failResolve) throw new Error('resolver down');
const sub = hosts.get(data.host);
if (sub) data.result.subdomain = sub;
};
const driverConfig = () =>
(driver as unknown as { config: Record<string, unknown> }).config;
beforeAll(() => {
driverConfig().custom_domains_enabled = true;
server.clients.event.on('site.resolveHost', resolveHost);
});
afterAll(() => {
delete driverConfig().custom_domains_enabled;
server.clients.event.off('site.resolveHost', resolveHost);
});
const attachDomain = async (userId: number) => {
const subdomain = uniqueName('cd');
await server.stores.subdomain.create({ userId, subdomain });
const host = `${uniqueName('www')}.custom-domain.test`;
hosts.set(host, subdomain);
return { host, subdomain };
};
it("refuses an index_url on another user's custom domain", async () => {
const owner = await makeUser();
const other = await makeUser();
const { host, subdomain } = await attachDomain(owner.userId);
const err = await withActor(other.actor, () =>
driver.create({
object: {
name: uniqueName('cd-squat'),
title: 't',
index_url: `https://${host}/`,
},
}),
).catch((e: unknown) => e);
expect(err).toMatchObject({
statusCode: 400,
legacyCode: 'subdomain_not_owned',
});
expect(JSON.stringify(err)).not.toContain(subdomain);
});
it("refuses repointing an app onto another user's custom domain", async () => {
const owner = await makeUser();
const other = await makeUser();
const { host } = await attachDomain(owner.userId);
const created = await withActor(other.actor, () =>
driver.create({
object: {
name: uniqueName('cd-repoint'),
title: 't',
index_url: uniqueIndexUrl(),
},
}),
);
await expect(
withActor(other.actor, () =>
driver.update({
uid: created.uid,
object: { index_url: `http://${host}/sub/` },
}),
),
).rejects.toMatchObject({
statusCode: 400,
legacyCode: 'subdomain_not_owned',
});
});
it("absorbs the owner's bootstrap row on create", async () => {
const { actor, userId } = await makeUser();
const { host } = await attachDomain(userId);
// AuthController stamps the site owner on the bootstrap row.
const stubUid = `app-${uuidv4()}`;
await server.stores.app.createFromOrigin(stubUid, `https://${host}`, {
ownerUserId: userId,
});
const name = uniqueName('cd-create');
const result = await withActor(actor, () =>
driver.create({
object: { name, title: 'Mine', index_url: `https://${host}/` },
}),
);
expect(result.uid).toBe(stubUid);
expect(result.name).toBe(name);
expect((await server.stores.app.getByUid(stubUid))?.owner_user_id).toBe(
userId,
);
});
it('keeps one app per custom-domain URL', async () => {
const { actor, userId } = await makeUser();
const { host } = await attachDomain(userId);
const first = await withActor(actor, () =>
driver.create({
object: {
name: uniqueName('cd-first'),
title: 't',
index_url: `https://${host}/`,
},
}),
);
await expect(
withActor(actor, () =>
driver.create({
object: {
name: uniqueName('cd-second'),
title: 't',
index_url: `http://${host}/index.html`,
},
}),
),
).rejects.toMatchObject({
statusCode: 400,
legacyCode: 'app_index_url_already_in_use',
fields: { app_uid: first.uid },
});
});
it('treats every host as external without a resolver (no prod extension)', async () => {
const a = await makeUser();
const b = await makeUser();
const url = uniqueIndexUrl();
server.clients.event.off('site.resolveHost', resolveHost);
try {
for (const { actor } of [a, b]) {
await withActor(actor, () =>
driver.create({
object: {
name: uniqueName('cd-none'),
title: 't',
index_url: url,
},
}),
);
}
} finally {
server.clients.event.on('site.resolveHost', resolveHost);
}
});
it('fails closed when the resolver errors', async () => {
const { actor } = await makeUser();
failResolve = true;
try {
await expect(
withActor(actor, () =>
driver.create({
object: {
name: uniqueName('cd-down'),
title: 't',
index_url: uniqueIndexUrl(),
},
}),
),
).rejects.toMatchObject({ statusCode: 503 });
} finally {
failResolve = false;
}
});
});
// ── hosted-subdomain ownership check ────────────────────────────────
//
// `#ensurePuterSiteSubdomainIsOwned` gates puter-hosted index_urls on a
@@ -45,8 +45,6 @@ import {
kvSharePermission,
type KvShareAccess,
} from '../../services/kvShare/kvShares.js';
import { claimCountedRateLimit } from '../../core/http/middleware/rateLimit.js';
import { kvNamespace } from '../../stores/systemKv/SystemKVStore.js';
import {
createTestUser,
setupPuterTestEnv,
@@ -56,7 +54,6 @@ import {
import type { IConfig } from '../../types.js';
import { openCursor, sealCursor } from '../../util/pagination.js';
import type { KVStoreDriver } from './KVStoreDriver.js';
import { KV_HANDLE_NAMESPACE_BUDGET } from './limits.js';
const BOOT_TIMEOUT_MS = 120_000;
const GLOBAL = 'os-global';
@@ -246,36 +243,6 @@ afterAll(async () => {
await env?.shutdown();
});
/**
* The handle budget calls into `owner`'s namespace spend from: every holder's,
* or with `holder`, that holder's own.
*/
const budgetKey = (owner: TestUser, holder?: TestUser, appUid = GLOBAL) => {
const namespace = `kv:handles:${kvNamespace(owner.uuid, appUid)}`;
return holder ? `${namespace}:holder:${holder.id}` : namespace;
};
/** A budget's counters for this window and the one before. */
const budgetCounters = (key: string) => {
const index = Math.floor(Date.now() / KV_HANDLE_NAMESPACE_BUDGET.window);
return [index - 1, index].map((at) => `ratec:{${key}}:${at}`);
};
/** Units spent from a budget since it was last cleared. */
const spent = async (key: string) => {
let total = 0;
for (const counter of budgetCounters(key))
total += Number(await env.server.clients.redis.get(counter));
return total;
};
const clearBudget = (key: string) =>
env.server.clients.redis.del(...budgetCounters(key));
/** Spend all of a budget, as though other calls had. */
const exhaust = (key: string, limit: number) =>
claimCountedRateLimit(key, limit, KV_HANDLE_NAMESPACE_BUDGET.window, limit);
// Every mint spends the 60/min budget, and this file makes more than that.
beforeEach(async () => {
for (const user of [owner, guest, stranger]) {
@@ -283,9 +250,7 @@ beforeEach(async () => {
`rate:${EVENTS_KV_HANDLE_LIMIT.scope}:${user.id}`,
);
await env.server.clients.redis.del(`rate:events:subscribe:${user.id}`);
await clearBudget(budgetKey(owner, user));
}
await clearBudget(budgetKey(owner));
});
describe('a call through a handle', () => {
@@ -445,17 +410,6 @@ describe('the level a call needs', () => {
legacyCode: 'kv_handle_access_denied',
});
});
it('keeps working after the owner stops a legacy share spelled in another case', async () => {
const tag = uuidv4().slice(0, 8);
const legacy = await legacyShare(`Notes-${tag}:`, 'read');
await ownerSet(`notes-${tag}:x:k`, 'v');
const { handle } = await share(`notes-${tag}:x:`, 'read');
await shares().removeKvShare(owner.actor, legacy.handle);
expect(await guestGet(handle, 'k')).toBe('v');
});
});
describe('a call refused before the handle is looked at', () => {
@@ -913,12 +867,11 @@ describe('listing through a handle', () => {
});
describe('a private entry in the region', () => {
it('is absent to the holder, uncounted, and gives nothing away to a write', async () => {
it('is absent to the holder, uncounted, and refuses every write', async () => {
const prefix = region('priv');
await ownerSet(`${prefix}open`, 1);
await ownerSet(`${prefix}secret`, 's', { disableSharing: true });
const { handle } = await share(prefix, 'write');
const { handle: readOnly } = await share(region('privro'), 'read');
expect(await guestGet(handle, 'secret')).toBeNull();
expect(await guestGet(handle, ['open', 'secret'])).toEqual([1, null]);
@@ -933,78 +886,18 @@ describe('a private entry in the region', () => {
),
).toEqual({ items: ['open'], total: 1 });
// Deleting it answers as deleting a key that isn't there.
expect(
await as(guest.actor, (d) =>
d.del({ key: 'secret', ...via(handle) }),
),
).toBe(true);
expect(
await as(guest.actor, (d) =>
d.del({ key: 'never-written', ...via(handle) }),
),
).toBe(true);
expect(
await as(guest.actor, (d) =>
d.remove({ key: 'secret', paths: [''], ...via(handle) }),
),
).toBeNull();
// Anything else is refused as a call the handle doesn't allow.
let levelRefusal: unknown;
try {
await guestSet(readOnly, 'k', 1);
} catch (err) {
levelRefusal = err;
}
const { statusCode, legacyCode, message } = levelRefusal as Record<
string,
unknown
>;
expect(legacyCode).toBe('kv_handle_access_denied');
for (const call of [
(d: KVStoreDriver) =>
d.set({ key: 'secret', value: 'x', ...via(handle) }),
(d: KVStoreDriver) =>
d.set({
key: 'secret',
value: 'x',
...via(handle, { onlyIfAbsent: true }),
}),
(d: KVStoreDriver) => d.del({ key: 'secret', ...via(handle) }),
(d: KVStoreDriver) =>
d.update({
key: 'secret',
pathAndValueMap: { a: 1 },
...via(handle),
}),
(d: KVStoreDriver) =>
d.add({
key: 'secret',
pathAndValueMap: { list: 1 },
...via(handle),
}),
(d: KVStoreDriver) =>
d.remove({ key: 'secret', paths: ['a'], ...via(handle) }),
(d: KVStoreDriver) =>
d.incr({
key: 'secret',
pathAndAmountMap: { n: 1 },
...via(handle),
}),
(d: KVStoreDriver) =>
d.decr({
key: 'secret',
pathAndAmountMap: { n: 1 },
...via(handle),
}),
(d: KVStoreDriver) =>
d.expire({ key: 'secret', ttl: 60, ...via(handle) }),
(d: KVStoreDriver) =>
d.expireAt({
key: 'secret',
timestamp: later(),
...via(handle),
}),
(d: KVStoreDriver) =>
d.batchPut({
items: [
@@ -1015,23 +908,18 @@ describe('a private entry in the region', () => {
}),
])
await expect(as(guest.actor, call)).rejects.toMatchObject({
statusCode,
legacyCode,
message,
statusCode: 403,
});
// A live key that isn't private still refuses `onlyIfAbsent` quietly.
expect(
await as(guest.actor, (d) =>
d.set({
key: 'open',
key: 'secret',
value: 'x',
...via(handle, { onlyIfAbsent: true }),
}),
),
).toBe(false);
expect(await ownerGet(`${prefix}secret`)).toBe('s');
expect(await ownerGet(`${prefix}open`)).toBe(1);
});
});
@@ -1287,44 +1175,6 @@ describe('what a write publishes', () => {
});
});
it('publishes a holder`s delete of a missing key, and nothing for a private one', async () => {
const prefix = region('pdel');
await ownerSet(`${prefix}secret`, 's', { disableSharing: true });
const { handle } = await share(prefix, 'write');
const ownerSub = (
await events().subscribe(owner.actor, SOCKET, {
subject: `kv:${GLOBAL}:${prefix}*`,
})
).sub;
const guestSub = (
await events().subscribe(guest.actor, SOCKET, {
subject: `kv:${handle}:*`,
})
).sub;
delivered.length = 0;
await as(guest.actor, (d) => d.del({ key: 'secret', ...via(handle) }));
await as(guest.actor, (d) => d.del({ key: 'missing', ...via(handle) }));
expect(await deliveryFor(guestSub.subId)).toMatchObject({
key: 'missing',
});
expect(await deliveryFor(ownerSub.subId)).toMatchObject({
key: `${prefix}missing`,
});
await new Promise((resolve) =>
setTimeout(resolve, EVENTS_COALESCE_WINDOW_MS * 2),
);
const keys = delivered
.filter((one) =>
[ownerSub.subId, guestSub.subId].includes(one.subId),
)
.map((one) => (one.event as { key?: string }).key);
expect(keys).not.toContain('secret');
expect(keys).not.toContain(`${prefix}secret`);
expect(await ownerGet(`${prefix}secret`)).toBe('s');
});
it('marks the owner`s own writes as theirs, and a worker`s or handler`s as via a worker', async () => {
const prefix = region('via');
const sub = (
@@ -1447,157 +1297,6 @@ describe('what a call costs', () => {
});
});
describe('the handle budget of a namespace', () => {
const { limit, perHolder } = KV_HANDLE_NAMESPACE_BUDGET;
it('is spent by every holder together, and refuses them all once it is gone', async () => {
const prefix = region('budget');
await ownerSet(`${prefix}k`, 'v');
const forGuest = await share(prefix, 'read');
const forStranger = await share(prefix, 'read', { holder: stranger });
expect(await guestGet(forGuest.handle, 'k')).toBe('v');
expect(await guestGet(forStranger.handle, 'k', stranger.actor)).toBe(
'v',
);
expect(await spent(budgetKey(owner))).toBe(2);
expect(await spent(budgetKey(owner, guest))).toBe(1);
await exhaust(budgetKey(owner), limit);
for (const call of [
() => guestGet(forGuest.handle, 'k'),
() => guestGet(forStranger.handle, 'k', stranger.actor),
])
await expect(call()).rejects.toMatchObject({
statusCode: 429,
legacyCode: 'too_many_requests',
});
// The owner's own calls spend none of it, with a handle or without.
expect(await ownerGet(`${prefix}k`)).toBe('v');
expect(await guestGet(forGuest.handle, 'k', owner.actor)).toBe('v');
// Another owner's namespace has its own.
const { handle: elsewhere } = await shares().mintKvHandle(
guest.actor,
{
prefix: region('budget-other'),
recipient: { username: owner.username },
access: 'read',
},
{ defaultAccess: 'read' },
);
expect(await guestGet(elsewhere, 'k', owner.actor)).toBeNull();
await clearBudget(budgetKey(owner));
expect(await guestGet(forGuest.handle, 'k')).toBe('v');
});
it('keeps one holder to its share, while the others get through', async () => {
const prefix = region('fair');
await ownerSet(`${prefix}k`, 'v');
const forGuest = await share(prefix, 'read');
const forStranger = await share(prefix, 'read', { holder: stranger });
expect(perHolder).toBeLessThan(limit);
await exhaust(budgetKey(owner, guest), perHolder);
await expect(guestGet(forGuest.handle, 'k')).rejects.toMatchObject({
statusCode: 429,
legacyCode: 'too_many_requests',
});
expect(await guestGet(forStranger.handle, 'k', stranger.actor)).toBe(
'v',
);
// The throttled holder's refusals spent nothing of the namespace's.
expect(await spent(budgetKey(owner))).toBe(1);
});
it('spends nothing when the owner calls through a handle', async () => {
const prefix = region('own');
await ownerSet(`${prefix}k`, 'v');
const { handle } = await share(prefix, 'read');
await exhaust(budgetKey(owner, owner), perHolder);
for (let i = 0; i < 3; i++)
expect(await guestGet(handle, 'k', owner.actor)).toBe('v');
expect(await spent(budgetKey(owner))).toBe(0);
await exhaust(budgetKey(owner), limit);
expect(await guestGet(handle, 'k', owner.actor)).toBe('v');
});
it('refuses a throttled holder before looking at its grant', async () => {
const { handle } = await share(region('early'), 'read');
await exhaust(budgetKey(owner, guest), perHolder);
const checks = vi.spyOn(env.server.services.permission, 'check');
try {
await expect(guestGet(handle, 'k')).rejects.toMatchObject({
statusCode: 429,
});
expect(checks).not.toHaveBeenCalled();
} finally {
checks.mockRestore();
}
});
it('is only heard by a caller the grant admits', async () => {
const { handle } = await share(region('heard'), 'watch');
await exhaust(budgetKey(owner), limit);
// Below the level: refused for that, not for the budget.
await expect(guestGet(handle, 'k')).rejects.toMatchObject({
legacyCode: 'kv_handle_access_denied',
});
});
it('tells another app of the holder the handle is not there, however spent', async () => {
const app = await makeApp();
const otherApp = await makeApp();
await authorizeApp(guest, app);
await authorizeApp(guest, otherApp);
const { handle } = await share(region('apps'), 'read', {
appUid: app.uid,
});
const holderBudget = budgetKey(owner, guest, app.uid);
await clearBudget(holderBudget);
await exhaust(holderBudget, perHolder);
await expect(
guestGet(handle, 'k', appActorFor(guest, otherApp)),
).rejects.toMatchObject({ legacyCode: 'kv_handle_not_found' });
await expect(
guestGet(handle, 'k', appActorFor(guest, app)),
).rejects.toMatchObject({ statusCode: 429 });
await clearBudget(holderBudget);
});
it('charges a call by the storage it reads and writes', async () => {
const prefix = region('cost');
const { handle } = await share(prefix, 'write');
const big = 'x'.repeat(100 * 1024);
for (const key of ['a', 'b', 'c', 'd'])
await ownerSet(`${prefix}big:${key}`, big);
const both = async () => [
await spent(budgetKey(owner)),
await spent(budgetKey(owner, guest)),
];
expect(await both()).toEqual([0, 0]);
await guestSet(handle, 'small', 'x');
await vi.waitFor(async () => expect(await both()).toEqual([3, 3]));
expect(await guestGet(handle, 'small')).toBe('x');
await vi.waitFor(async () => expect(await both()).toEqual([4, 4]));
const page = (await as(guest.actor, (d) =>
d.list({ as: 'entries', pattern: 'big:*', ...via(handle) }),
)) as unknown[];
expect(page).toHaveLength(4);
// About 400 KB read, at a unit per 8 KB.
await vi.waitFor(async () => {
for (const units of await both()) {
expect(units).toBeGreaterThanOrEqual(4 + 50);
expect(units).toBeLessThan(4 + 60);
}
});
});
});
describe('what a holder is ever shown', () => {
it('names no owner, prefix or whole key in any answer, success or error', async () => {
const prefix = region('scan');
+20 -125
View File
@@ -19,13 +19,12 @@
import { HttpError } from '../../core/http/HttpError.js';
import { Context } from '../../core/context.js';
import { claimCountedRateLimit } from '../../core/http/middleware/rateLimit.js';
import {
DEFAULT_FREE_SUBSCRIPTION,
DEFAULT_TEMP_SUBSCRIPTION,
} from '../../services/metering/consts.js';
import { PuterDriver } from '../types.js';
import { type Actor, isAccountContext, isAppActor } from '../../core/actor.js';
import type { Actor } from '../../core/actor.js';
import type { DriverConcurrentConfig, DriverRateLimitConfig } from '../meta.js';
import {
APP_DATA_KV_METHOD_OPS,
@@ -36,7 +35,6 @@ import {
import {
KV_GLOBAL_APP_KEY,
KV_MAX_KEY_BYTES,
KvPrivateEntryError,
kvNamespace,
type KVOpts,
type KVUsage,
@@ -57,11 +55,8 @@ import {
sealCursor,
unsealCursor,
} from '../../util/pagination.js';
import { predatesApp } from '../../util/appGeneration.js';
import { KV_COSTS } from './costs.js';
import {
KV_HANDLE_NAMESPACE_BUDGET,
KV_HANDLE_WRITE_UNIT_WEIGHT,
KV_MAX_BATCH_PUT_ITEMS,
KV_MAX_GET_KEYS,
KV_MAX_LIST_LIMIT,
@@ -98,20 +93,7 @@ type KvCallArgs = {
* Store options plus, for a call through a share handle, the handle and its
* region's prefix: keys go in under it and come back out relative to it.
*/
type DriverKvOpts = KVOpts & {
handle?: string;
handlePrefix?: string;
/** Set when the call spends from a namespace's handle budget. */
handleBudget?: HandleBudget;
};
/** The budgets a holder's call through a handle spends from. */
interface HandleBudget {
/** Every holder of a share into the namespace together. */
namespace: string;
/** This holder alone, within the namespace. */
holder: string;
}
type DriverKvOpts = KVOpts & { handle?: string; handlePrefix?: string };
type KvListResult =
unknown[] | { items: unknown[]; cursor?: string; total?: number };
@@ -182,43 +164,6 @@ const handlePatternInvalid = (): HttpError =>
{ legacyCode: 'kv_handle_key_invalid' },
);
/** The holder's budget in the namespace, or the namespace's, is used up. */
const handleBudgetSpent = (): HttpError =>
new HttpError(429, 'Too many requests.', {
legacyCode: 'too_many_requests',
});
/** The budgets a holder's calls into `namespace` spend from. */
const handleBudget = (
namespace: string,
holderUserId: number,
): HandleBudget => ({
namespace: `kv:handles:${namespace}`,
holder: `kv:handles:${namespace}:holder:${holderUserId}`,
});
/** Spend `units` from a budget, and say whether any of them fit. */
const spendHandleBudget = async (
key: string,
limit: number,
units: number,
): Promise<boolean> =>
(await claimCountedRateLimit(
key,
limit,
KV_HANDLE_NAMESPACE_BUDGET.window,
units,
)) > 0;
/**
* Whether a share's grant can answer for this actor at all: the holder's own
* account, or the app whose data is shared. Any other caller is refused as if
* the handle weren't there.
*/
const onShareSurface = (actor: Actor, appUid: string): boolean =>
isAccountContext(actor) ||
(isAppActor(actor) && actor.effectiveApp?.uid === appUid);
const invalidCursor = (): HttpError =>
new HttpError(400, 'invalid kv: cursor', { legacyCode: 'bad_request' });
@@ -386,8 +331,7 @@ export class KVStoreDriver extends PuterDriver {
/**
* Run one store call. Through a handle, an error naming a stored key names
* the caller's relative key instead, and a write refused for a private
* entry gets the refusal any call the handle doesn't allow gets.
* the caller's relative key instead.
*/
async #inRegion<T>(
opts: DriverKvOpts,
@@ -398,7 +342,6 @@ export class KVStoreDriver extends PuterDriver {
try {
return await call();
} catch (err) {
if (err instanceof KvPrivateEntryError) throw handleAccessDenied();
throw relativizeError(err, opts.handlePrefix, keys);
}
}
@@ -492,31 +435,11 @@ export class KVStoreDriver extends PuterDriver {
if (userId !== row.granteeUserId && userId !== row.ownerUserId)
throw handleNotFound();
// The holder's own budget goes before the grant lookups, so throttled
// retries cost none. It only counts the holder's own calls, so it says
// nothing about the namespace. The owner spends no handle budget.
const { limit, perHolder } = KV_HANDLE_NAMESPACE_BUDGET;
const budget =
userId === row.ownerUserId
? undefined
: handleBudget(kvNamespace(ownerUuid, row.appUid), userId);
const holderFirst =
budget !== undefined && onShareSurface(actor, row.appUid);
if (
holderFirst &&
!(await spendHandleBudget(budget.holder, perHolder, 1))
)
throw handleBudgetSpent();
// Side by side, so the app lookup costs no round trip of its own. A
// handle older than its app was made for an earlier app under the
// same uid, and is gone with it.
// Side by side, so the app lookup costs no round trip of its own.
const [appGone, allowed] = await Promise.all([
row.appUid === KV_GLOBAL_APP_KEY
? false
: this.stores.app
.getByUid(row.appUid)
.then((app) => !app || predatesApp(row.createdAt, app)),
: this.stores.app.getByUid(row.appUid).then((app) => !app),
this.services.permission.check(
actor,
kvSharePermission(ownerUuid, row.appUid, row.keyPrefix, level),
@@ -543,17 +466,6 @@ export class KVStoreDriver extends PuterDriver {
Buffer.byteLength(row.keyPrefix, 'utf8') > KV_SHARE_PREFIX_MAX_BYTES
)
throw handleAccessDenied();
// The namespace's budget tells how busy other holders are, so only a
// caller the grant admits may hear it. `#meter` spends the rest.
if (budget) {
if (
!holderFirst &&
!(await spendHandleBudget(budget.holder, perHolder, 1))
)
throw handleBudgetSpent();
if (!(await spendHandleBudget(budget.namespace, limit, 1)))
throw handleBudgetSpent();
}
return {
actor,
@@ -561,7 +473,6 @@ export class KVStoreDriver extends PuterDriver {
namespaceAppUuid: row.appUid,
handle,
handlePrefix: row.keyPrefix,
...(budget ? { handleBudget: budget } : {}),
};
}
@@ -761,23 +672,7 @@ export class KVStoreDriver extends PuterDriver {
}
}
/**
* Bill the caller for what the call consumed. A holder's call through a
* handle also spends it from its budgets, past the unit admission took.
*/
#meter(opts: DriverKvOpts, usage: KVUsage): void {
const budget = opts.handleBudget;
if (budget) {
const units = Math.ceil(
usage.read + usage.write * KV_HANDLE_WRITE_UNIT_WEIGHT,
);
if (units > 1) {
const { limit, perHolder } = KV_HANDLE_NAMESPACE_BUDGET;
void spendHandleBudget(budget.holder, perHolder, units - 1);
void spendHandleBudget(budget.namespace, limit, units - 1);
}
}
const actor = opts.actor;
#meter(actor: Actor | undefined, usage: KVUsage): void {
if (!actor) return;
const metering = this.services.metering;
if (usage.read > 0) {
@@ -855,7 +750,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -863,7 +758,7 @@ export class KVStoreDriver extends PuterDriver {
const { res, usage } = await this.#inRegion(opts, [coerced], () =>
this.stores.kv.get({ key: this.#storeKey(coerced, opts) }, opts),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -898,7 +793,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -964,7 +859,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -977,7 +872,7 @@ export class KVStoreDriver extends PuterDriver {
const { res, usage } = await this.#inRegion(opts, [coerced], () =>
this.stores.kv.del({ key: this.#storeKey(coerced, opts) }, opts),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1033,7 +928,7 @@ export class KVStoreDriver extends PuterDriver {
},
opts,
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
if (!listing) return res;
return this.#foreignResult(res as KvListResult, args.as, listing);
}
@@ -1041,7 +936,7 @@ export class KVStoreDriver extends PuterDriver {
async flush(args: { optConfig?: KvOptConfig }): Promise<boolean> {
const opts = await this.#opts('flush', args);
const { res, usage } = await this.stores.kv.flush(opts);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1069,7 +964,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1097,7 +992,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1122,7 +1017,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1144,7 +1039,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1171,7 +1066,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1196,7 +1091,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
return res;
}
@@ -1218,7 +1113,7 @@ export class KVStoreDriver extends PuterDriver {
opts,
),
);
this.#meter(opts, usage);
this.#meter(opts.actor, usage);
// The result is the entry's value, so a delete-only grant gets none. A
// handle's `write` level reads too.
if (
+2 -22
View File
@@ -17,8 +17,8 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// Caps on the public KV driver. Enforced in the driver rather than the store,
// whose internal callers batch past them.
// Per-call caps on the public KV driver. Enforced here rather than in the
// store, whose internal callers batch past them.
/** Keys one array `get` may name. */
export const KV_MAX_GET_KEYS = 1000;
@@ -31,23 +31,3 @@ export const KV_MAX_BATCH_PUT_ITEMS = 1000;
* the cursor carries the rest. Matches the SDK's own page size.
*/
export const KV_MAX_LIST_LIMIT = 1000;
/**
* Storage capacity calls through share handles may spend in one owner's
* namespace per window: `limit` for every holder together, and `perHolder` for
* any one of them, so no holder can starve the others. The owner's own calls
* through a handle spend none. Sized to leave about half of what one namespace
* can serve to the owner's own app.
*/
export const KV_HANDLE_NAMESPACE_BUDGET = {
limit: 15_000,
perHolder: 3_750,
window: 10_000,
};
/**
* What a call spends from those budgets: its read units, plus this many per
* write unit, since a write costs the store about three times what a read does.
* Every call spends at least one.
*/
export const KV_HANDLE_WRITE_UNIT_WEIGHT = 3;
@@ -145,18 +145,6 @@ const captureBus = async (
return seen;
};
const DAY_MS = 24 * 60 * 60_000;
/** Exactly `count` slots of a window are spent. */
const expectSpent = async (
key: string,
windowMs: number,
count: number,
): Promise<void> => {
expect(await peekRateLimit(key, count, windowMs)).toBe(false);
expect(await peekRateLimit(key, count + 1, windowMs)).toBe(true);
};
const bootServer = (config: Partial<IConfig>) => {
const state = {
server: null as unknown as PuterServer,
@@ -183,19 +171,16 @@ describe('app.message with live delivery over the socket wire', () => {
const app = await makeApp(state.server, holder.user.id!);
let created: Record<string, unknown> | null = null;
let desktop: Array<Record<string, unknown>> = [];
const written = await captureBus(state.server, 'notif.written', () =>
captureBus(state.server, 'outer.gui.notif.message', async () => {
const pushed = await captureBus(
state.server,
'outer.gui.notif.message',
async () => {
created = (await withActor(asApp(holder, app), () =>
state.driver.create(
message({
text: 'Body',
priority: 'high',
channel: 'replies',
}),
message({ text: 'Body', channel: 'replies' }),
),
)) as Record<string, unknown>;
}).then((seen) => (desktop = seen)),
},
);
const uid = created!.uid as string;
@@ -204,7 +189,7 @@ describe('app.message with live delivery over the socket wire', () => {
title: 'Hello',
text: 'Body',
channel: 'replies',
priority: 'high',
priority: 'normal',
ttl: 3600,
app: { uid: app.uid, name: 'App message test' },
});
@@ -216,18 +201,14 @@ describe('app.message with live delivery over the socket wire', () => {
app_uid: app.uid,
audience: 'app-user',
type: 'app.message',
value: expect.objectContaining({ priority: 'high' }),
});
// Not silent, but sent for the app alone: the desktop wire stays quiet.
expect(written).toEqual([
expect.objectContaining({
userId: holder.user.id,
uid,
type: 'app.message',
appUid: app.uid,
value: expect.objectContaining({ priority: 'high' }), }),
// Not silent: it went out on the wire the desktop listens to.
expect(pushed).toEqual([
{
user_id_list: [holder.user.id],
response: { uid, notification: created!.value },
},
]);
expect(desktop).toEqual([]);
});
it("lets the app's worker and the tokens it issued notify their user", async () => {
@@ -319,42 +300,6 @@ describe('app.message with live delivery over the socket wire', () => {
).toEqual([]);
});
it('names the field over its own limit before the total size', async () => {
const holder = await makeUser(state.server);
const app = await makeApp(state.server, holder.user.id!);
for (const [fields, field] of [
[{ data: { blob: 'x'.repeat(20_000) } }, 'data'],
[{ text: 'x'.repeat(20_000) }, 'text'],
[{ extra: 'x'.repeat(20_000) }, 'extra'],
] as const) {
await expect(
withActor(asApp(holder, app), () =>
state.driver.create(message(fields)),
),
).rejects.toMatchObject({
statusCode: 400,
legacyCode: 'bad_request',
fields: { field },
});
}
});
it('still caps the whole value when every field passes', async () => {
const holder = await makeUser(state.server);
const app = await makeApp(state.server, holder.user.id!);
// Invisible marks are dropped from the title, so only the total is over.
await expect(
withActor(asApp(holder, app), () =>
state.driver.create(
message({ title: `${'\u200b'.repeat(20_000)}Hi` }),
),
),
).rejects.toMatchObject({
statusCode: 400,
legacyCode: 'notification_value_too_large',
});
});
it('keeps the holder creating other types silently', async () => {
const holder = await makeUser(state.server);
const pushed = await captureBus(
@@ -520,176 +465,6 @@ describe('app.message with live delivery over the socket wire', () => {
});
});
describe('app.message groups', () => {
const state = bootServer({
notificationAppMessageLimits: { perMinute: 5, perDay: 4 },
});
const setup = async () => {
const holder = await makeUser(state.server);
const app = await makeApp(state.server, holder.user.id!);
const send = async (fields: Record<string, unknown> = {}) =>
(await withActor(asApp(holder, app), () =>
state.driver.create(message(fields)),
)) as Record<string, unknown>;
const rows = () =>
state.server.stores.notification.listByUserId(holder.user.id!, {});
return {
holder,
app,
send,
rows,
minuteKey: `notif:app-message:minute:${holder.user.id}:${app.uid}`,
dayKey: `notif:app-message:day:${holder.user.id}:${app.uid}`,
};
};
it('replaces the unseen message with the same group, spending no day slot', async () => {
const { holder, app, send, rows, minuteKey, dayKey } = await setup();
const first = await send({ title: 'One', group: 'chat-1' });
let second: Record<string, unknown> = {};
const written = await captureBus(
state.server,
'notif.written',
async () => {
second = await send({
title: 'Two',
text: 'Hi',
group: 'chat-1',
});
},
);
expect(second.uid).toBe(first.uid);
expect(second.value).toMatchObject({ title: 'Two', text: 'Hi' });
expect(await rows()).toEqual([
expect.objectContaining({
uid: first.uid,
value: expect.objectContaining({ title: 'Two' }),
}),
]);
expect(written).toEqual([
expect.objectContaining({
userId: holder.user.id,
uid: first.uid,
appUid: app.uid,
updated: true,
}),
]);
await expectSpent(minuteKey, 60_000, 2);
await expectSpent(dayKey, DAY_MS, 1);
});
it('starts a new message once the last one was shown or dismissed', async () => {
const { holder, app, send, rows, dayKey } = await setup();
const asTheApp = asApp(holder, app);
const first = await send({ group: 'chat-1' });
await withActor(asTheApp, () =>
state.driver.mark_shown({ uid: first.uid }),
);
const second = await send({ group: 'chat-1' });
expect(second.uid).not.toBe(first.uid);
await withActor(asTheApp, () =>
state.driver.mark_acknowledged({ uid: second.uid }),
);
const third = await send({ group: 'chat-1' });
expect(third.uid).not.toBe(second.uid);
expect(await rows()).toHaveLength(3);
await expectSpent(dayKey, DAY_MS, 3);
});
it('keeps messages with no group, or another group, apart', async () => {
const { send, rows } = await setup();
const uids = [
(await send()).uid,
(await send()).uid,
(await send({ group: 'a' })).uid,
(await send({ group: 'b' })).uid,
];
expect(new Set(uids).size).toBe(4);
expect(await rows()).toHaveLength(4);
});
it('still spends the minute window on a replacement', async () => {
const { send, rows, dayKey } = await setup();
for (let i = 0; i < 5; i++) await send({ group: 'chat-1' });
await expect(send({ group: 'chat-1' })).rejects.toMatchObject({
statusCode: 429,
legacyCode: 'too_many_requests',
});
expect(await rows()).toHaveLength(1);
await expectSpent(dayKey, DAY_MS, 1);
});
it('replaces with the day window spent', async () => {
const { send, rows } = await setup();
const grouped = await send({ group: 'chat-1' });
for (let i = 0; i < 3; i++) await send();
await expect(send()).rejects.toMatchObject({ statusCode: 429 });
const replaced = await send({ title: 'Later', group: 'chat-1' });
expect(replaced.uid).toBe(grouped.uid);
expect(await rows()).toHaveLength(4);
});
it('writes a new message when the old one is shown before the replace lands', async () => {
const { holder, app, send, rows, minuteKey, dayKey } = await setup();
const first = await send({ group: 'chat-1' });
charges.held.add(minuteKey);
const pending = send({ title: 'Raced', group: 'chat-1' });
let second: Record<string, unknown>;
try {
await until(() => charges.waiting.length === 1);
await withActor(asApp(holder, app), () =>
state.driver.mark_shown({ uid: first.uid }),
);
} finally {
charges.held.clear();
charges.waiting.shift()?.();
second = await pending;
}
expect(second.uid).not.toBe(first.uid);
expect(await rows()).toHaveLength(2);
await expectSpent(dayKey, DAY_MS, 2);
});
it('folds concurrent messages to an empty group into one, spending one day slot', async () => {
const { send, rows, minuteKey, dayKey } = await setup();
const sent = await Promise.all(
Array.from({ length: 4 }, (_, i) =>
send({ title: `Message ${i}`, group: 'chat-1' }),
),
);
expect(new Set(sent.map((s) => s.uid)).size).toBe(1);
expect(await rows()).toHaveLength(1);
await expectSpent(minuteKey, 60_000, 4);
await expectSpent(dayKey, DAY_MS, 1);
});
it('still sends when the group stays locked past the wait', async () => {
const { holder, app, send, rows, dayKey } = await setup();
await state.server.clients.redis.set(
`notif:app-message:group:${holder.user.id}:${app.uid}:chat-1`,
'stalled-sender',
'PX',
60_000,
);
const sent = await send({ group: 'chat-1' });
expect(await rows()).toEqual([
expect.objectContaining({ uid: sent.uid }),
]);
await expectSpent(dayKey, DAY_MS, 1);
});
});
describe('app.message with the events fold-in on', () => {
const state = bootServer({
events: { enabled: true, notificationsFoldIn: true },
@@ -728,34 +503,6 @@ describe('app.message with the events fold-in on', () => {
).toBe(`notif:${app.uid}:app-user`);
});
it('dispatches a replacement again under the same uid', async () => {
const holder = await makeUser(state.server);
const app = await makeApp(state.server, holder.user.id!);
const first = (await withActor(asApp(holder, app), () =>
state.driver.create(message({ group: 'g' })),
)) as Record<string, unknown>;
const created = await captureBus(state.server, 'notif.created', () =>
withActor(asApp(holder, app), () =>
state.driver.create(message({ title: 'Again', group: 'g' })),
),
);
expect(created).toEqual([
expect.objectContaining({
uid: first.uid,
appUid: app.uid,
audience: 'app-user',
value: expect.objectContaining({ title: 'Again' }),
replaced: true,
}),
]);
const row = await state.server.stores.notification.getByUid(
first.uid as string,
{ userId: holder.user.id, primary: true },
);
expect(Number(row.replaced_at)).toBe(created[0].createdAt);
});
it('limits each app per user per day, without spending the minute', async () => {
const holder = await makeUser(state.server);
const app = await makeApp(state.server, holder.user.id!);
@@ -1,438 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/** `app.message` sent to the other party of a KV share through its handle. */
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import {
EVENTS_KV_HANDLE_LIMIT,
KV_SHARE_NOTIFY_LIMITS,
} from '../../controllers/events/limits.js';
import { makeActor, type Actor } from '../../core/actor.js';
import { runWithContext } from '../../core/context.js';
import {
claimRateLimit,
peekRateLimit,
} from '../../core/http/middleware/rateLimit.js';
import {
createTestUser,
setupPuterTestEnv,
type PuterTestEnv,
} from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import type { NotificationDriver } from './NotificationDriver.js';
const BOOT_TIMEOUT_MS = 120_000;
const DAY_MS = 24 * 60 * 60_000;
interface TestUser {
actor: Actor;
username: string;
id: number;
}
let env: PuterTestEnv;
let owner: TestUser;
let guest: TestUser;
let stranger: TestUser;
const driver = () =>
env.server.drivers.notifications as unknown as NotificationDriver;
const userFor = async (username: string): Promise<TestUser> => {
const row = await env.server.stores.user.getByUsername(username);
return {
actor: makeActor({ user: row as never }),
username,
id: row!.id!,
};
};
const makeApp = async () => {
const name = `notify-to-${uuidv4().slice(0, 8)}`;
return env.server.stores.app.create(
{
name,
title: 'Handle App',
index_url: `https://${name}.example.test/index.html`,
},
{ ownerUserId: owner.id },
) as Promise<{ uid: string; id: number }>;
};
const inApp = (user: TestUser, app: { uid: string; id: number }) =>
makeActor({
user: user.actor.user as never,
app: { uid: app.uid, id: app.id },
});
/** A share of `app`'s data from the owner to the guest. */
const share = async (app: { uid: string }, notify = true) =>
env.server.services.kvShare.mintKvHandle(
owner.actor,
{
prefix: `room-${uuidv4().slice(0, 8)}:`,
recipient: { username: guest.username },
appUid: app.uid,
notify,
},
{ defaultAccess: 'write' },
);
const send = (actor: Actor, fields: Record<string, unknown> = {}) =>
runWithContext({ actor }, () =>
driver().create({
object: { value: { type: 'app.message', title: 'Hi', ...fields } },
}),
) as Promise<Record<string, unknown>>;
const rowsOf = (user: TestUser) =>
env.server.stores.notification.listByUserId(user.id, {});
/** The user's `app.message` rows from `app`, newest first. */
const appMessagesOf = async (user: TestUser, app: { uid: string }) =>
(await rowsOf(user)).filter(
(row: Record<string, unknown>) =>
row.app_uid === app.uid && row.type === 'app.message',
);
/** What a refusal shows the sender. */
const refusalOf = async (sending: Promise<unknown>) => {
try {
await sending;
} catch (err) {
const { statusCode, legacyCode, message, fields } = err as Record<
string,
unknown
>;
return { statusCode, legacyCode, message, fields };
}
throw new Error('expected a refusal');
};
const tooMany = {
statusCode: 429,
legacyCode: 'too_many_requests',
message: 'Too many notifications.',
};
const minuteKey = (user: TestUser, appUid: string) =>
`notif:app-message:minute:${user.id}:${appUid}`;
const dayKey = (user: TestUser, appUid: string) =>
`notif:app-message:day:${user.id}:${appUid}`;
/** Exactly `count` slots of a window are spent. */
const expectSpent = async (key: string, windowMs: number, count: number) => {
if (count > 0)
expect(await peekRateLimit(key, count, windowMs)).toBe(false);
expect(await peekRateLimit(key, count + 1, windowMs)).toBe(true);
};
/** Every payload the bus carried under `key` while `fn` ran. */
const captureBus = async (
key: string,
fn: () => Promise<unknown>,
): Promise<Array<Record<string, unknown>>> => {
const seen: Array<Record<string, unknown>> = [];
const handler = (_k: string, data: unknown) =>
seen.push(data as Record<string, unknown>);
env.server.clients.event.on(key, handler);
try {
await fn();
await new Promise((resolve) => setTimeout(resolve, 20));
} finally {
env.server.clients.event.off?.(key, handler);
}
return seen;
};
beforeAll(async () => {
env = await setupPuterTestEnv({
events: { enabled: true, kvHandles: true },
kv: { shareHandles: true },
unlimitedMetering: true,
notificationAppMessageLimits: { perMinute: 5, perDay: 4 },
} as IConfig);
const strangerName = `notify-to-${uuidv4().slice(0, 8)}`;
await createTestUser(env.server, {
username: strangerName,
password: 'pw-test-1234',
});
owner = await userFor(env.users.user.username);
guest = await userFor(env.users.other.username);
stranger = await userFor(strangerName);
}, BOOT_TIMEOUT_MS);
afterAll(async () => {
await env?.shutdown();
});
beforeEach(async () => {
await env.server.clients.redis.del(
`rate:${EVENTS_KV_HANDLE_LIMIT.scope}:${owner.id}`,
);
});
describe('app.message to the other party of a share', () => {
it('writes the owner’s message to the holder, and answers with nothing', async () => {
const app = await makeApp();
const { handle } = await share(app);
let sent: Record<string, unknown> | null = null;
const written = await captureBus('notif.written', async () => {
sent = await send(inApp(owner, app), {
title: 'New message',
priority: 'high',
to: { handle },
});
});
expect(sent).toEqual({});
const [row] = await appMessagesOf(guest, app);
expect(row).toMatchObject({
user_id: guest.id,
app_uid: app.uid,
type: 'app.message',
audience: 'app-user',
});
expect(row!.value).toEqual({
type: 'app.message',
title: 'New message',
priority: 'high',
ttl: 3600,
app: { uid: app.uid, name: 'Handle App' },
});
// The push extension bills and checks consent on this recipient.
expect(written).toEqual([
expect.objectContaining({
userId: guest.id,
uid: row.uid,
appUid: app.uid,
}),
]);
});
it('writes the holder’s message to the owner', async () => {
const app = await makeApp();
const { handle } = await share(app);
const before = (await rowsOf(guest)).length;
const sent = await send(inApp(guest, app), {
title: 'From the holder',
to: { handle },
});
expect(sent).toEqual({});
const [row] = await appMessagesOf(owner, app);
expect(row).toMatchObject({ user_id: owner.id });
expect(row.value).toMatchObject({ title: 'From the holder' });
expect(await rowsOf(guest)).toHaveLength(before);
});
it("charges the recipient's limits, not the sender's", async () => {
const app = await makeApp();
const { handle } = await share(app);
await send(inApp(owner, app), { to: { handle } });
await expectSpent(minuteKey(guest, app.uid), 60_000, 1);
await expectSpent(dayKey(guest, app.uid), DAY_MS, 1);
await expectSpent(minuteKey(owner, app.uid), 60_000, 0);
await expectSpent(dayKey(owner, app.uid), DAY_MS, 0);
});
it('answers the pair limit and the recipient’s limit the same way', async () => {
const app = await makeApp();
const { handle } = await share(app);
// The recipient's own day window runs out.
for (let i = 0; i < 4; i++)
await send(inApp(guest, app), { title: `own ${i}` });
const byRecipient = await refusalOf(
send(inApp(owner, app), { to: { handle } }),
);
const other = await makeApp();
const { handle: otherHandle } = await share(other);
const [minute] = KV_SHARE_NOTIFY_LIMITS;
await claimRateLimit(
`${minute.scope}:${owner.id}:${guest.id}:${other.uid}`,
minute.limit,
minute.window,
minute.limit,
);
const byPair = await refusalOf(
send(inApp(owner, other), { to: { handle: otherHandle } }),
);
expect(byRecipient).toEqual({ ...tooMany, fields: undefined });
expect(byPair).toEqual(byRecipient);
await expectSpent(dayKey(guest, other.uid), DAY_MS, 0);
});
it("replaces the recipient's newest message in the same group", async () => {
const app = await makeApp();
const { handle } = await share(app);
await send(inApp(owner, app), {
title: 'One',
group: 'room-1',
to: { handle },
});
const second = await send(inApp(owner, app), {
title: 'Two',
group: 'room-1',
to: { handle },
});
expect(second).toEqual({});
const rows = await appMessagesOf(guest, app);
expect(rows).toHaveLength(1);
expect(rows[0].value).toMatchObject({ title: 'Two' });
await expectSpent(dayKey(guest, app.uid), DAY_MS, 1);
});
it('folds concurrent messages to one group into one, through the share or not', async () => {
const app = await makeApp();
const { handle } = await share(app);
await Promise.all([
send(inApp(owner, app), { group: 'room-1', to: { handle } }),
send(inApp(owner, app), { group: 'room-1', to: { handle } }),
send(inApp(guest, app), { group: 'room-1' }),
send(inApp(guest, app), { group: 'room-1' }),
]);
expect(await appMessagesOf(guest, app)).toHaveLength(1);
await expectSpent(dayKey(guest, app.uid), DAY_MS, 1);
});
it.each([
['a string', 'kvh-1'],
['an array', [{ handle: 'kvh-1' }]],
['null', null],
['no handle', {}],
['a handle that is not a string', { handle: 7 }],
['an empty handle', { handle: '' }],
['another field beside the handle', { handle: 'kvh-1', user: 'x' }],
])('refuses `to` as %s', async (_label, to) => {
const app = await makeApp();
await expect(send(inApp(owner, app), { to })).rejects.toMatchObject({
statusCode: 400,
legacyCode: 'bad_request',
fields: { field: 'to' },
});
});
it('validates the message before spending the pair limit', async () => {
const app = await makeApp();
const { handle } = await share(app);
const [minute] = KV_SHARE_NOTIFY_LIMITS;
await expect(
send(inApp(owner, app), {
icon: 'http://insecure.test/a.png',
to: { handle },
}),
).rejects.toMatchObject({ statusCode: 400, fields: { field: 'icon' } });
expect(
await env.server.clients.redis.zcard(
`rate:${minute.scope}:${owner.id}:${guest.id}:${app.uid}`,
),
).toBe(0);
});
describe('passes every refusal of the share through with its code', () => {
const absent = {
statusCode: 404,
legacyCode: 'subject_does_not_exist',
};
it('an unknown handle, or one the sender is no party to', async () => {
const app = await makeApp();
const { handle } = await share(app);
await expect(
send(inApp(owner, app), { to: { handle: `kvh-${uuidv4()}` } }),
).rejects.toMatchObject(absent);
await expect(
send(inApp(stranger, app), { to: { handle } }),
).rejects.toMatchObject(absent);
await expect(
send(inApp(owner, await makeApp()), { to: { handle } }),
).rejects.toMatchObject(absent);
});
it('a share without notify', async () => {
const app = await makeApp();
const { handle } = await share(app, false);
await expect(
send(inApp(owner, app), { to: { handle } }),
).rejects.toMatchObject({
statusCode: 403,
legacyCode: 'kv_share_notify_not_enabled',
});
});
it('a recipient who blocked the sender', async () => {
const app = await makeApp();
const { handle } = await share(app);
await env.server.stores.userBlock.create(guest.id, owner.id);
try {
await expect(
send(inApp(owner, app), { to: { handle } }),
).rejects.toMatchObject({
statusCode: 403,
legacyCode: 'recipient_not_accepting_shares',
});
} finally {
await env.server.stores.userBlock.deleteByPair(
guest.id,
owner.id,
);
}
});
it('a share that cannot be checked', async () => {
const app = await makeApp();
const { handle } = await share(app);
const lookup = vi
.spyOn(env.server.stores.kvShareHandle, 'getLive')
.mockRejectedValue(new Error('lookup unavailable'));
try {
await expect(
send(inApp(owner, app), { to: { handle } }),
).rejects.toMatchObject({
statusCode: 503,
legacyCode: 'service_unavailable',
});
} finally {
lookup.mockRestore();
}
});
});
});
@@ -1,506 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* `notify({ sendAt })` without and with `services.schedules`, and the deferred
* mailbox write an extension runs inside its own transaction, fenced on a guard
* in a table of its own (a test table here).
*/
import { v4 as uuidv4 } from 'uuid';
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from 'vitest';
import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
import { makeActor, type Actor } from '../../core/actor.js';
import { runWithContext } from '../../core/context.js';
import { isHttpError } from '../../core/http/HttpError.js';
import type {
DeferredSendPlan,
ScheduleServiceContract,
} from '../../core/schedules.js';
import type { PuterServer } from '../../server.js';
import { EVENTS_BACKGROUND_PERMISSION } from '../../services/events/authorization.js';
import { setupTestServer } from '../../testUtil.js';
import type { IConfig } from '../../types.js';
import type {
DeferredWriteTx,
NotificationDriver,
} from './NotificationDriver.js';
vi.setConfig({ testTimeout: 120_000 });
const BOOT_TIMEOUT_MS = 480_000;
const TEST_TABLE = 'deferred_send_test';
let server: PuterServer;
let holder: Actor;
let app: { id: number; uid: string };
let saved: unknown;
let nextId = 1;
const db = () => server.clients.db;
const driver = () =>
server.drivers.notifications as unknown as NotificationDriver;
const slot = () => server.services as unknown as { schedules?: unknown };
/** Put `schedules` in the slot, or leave the slot empty. */
const setSchedules = (schedules: unknown): void => {
if (schedules === undefined) delete slot().schedules;
else slot().schedules = schedules;
};
const asApp = (): Actor => makeActor({ user: holder.user, app });
const notify = (fields: Record<string, unknown>): Promise<unknown> =>
runWithContext({ actor: asApp() }, () =>
driver().create({
object: { value: { type: 'app.message', title: 'Hi', ...fields } },
}),
);
const codeOf = async (run: Promise<unknown>): Promise<string | undefined> => {
try {
await run;
} catch (err) {
if (isHttpError(err)) return err.legacyCode;
throw err;
}
return undefined;
};
const mailbox = () =>
db().read(
'SELECT * FROM `notification` WHERE `user_id` = ? AND `app_uid` = ? ' +
'ORDER BY `id`',
[holder.user!.id, app.uid],
);
/** A row's value: JSON text on sqlite, parsed on the other engines. */
const valueOf = (row: Record<string, unknown>) =>
typeof row.value === 'string' ? JSON.parse(row.value) : row.value;
/** A send claimed under `token`, as an extension's own row would be. */
const claimed = async (token: string): Promise<number> => {
const id = nextId++;
await db().write(
`INSERT INTO \`${TEST_TABLE}\` (\`id\`, \`token\`, \`state\`) VALUES (?, ?, 'sending')`,
[id, token],
);
return id;
};
/** The transaction an extension wraps the write in: its row, then the mailbox. */
const txFor = (id: number, token: string): DeferredWriteTx => ({
guard: {
table: TEST_TABLE,
where: "`id` = ? AND `token` = ? AND `state` = 'delivered'",
values: [id, token],
},
async commit(entries: readonly BatchEntry[]) {
await db().batchWrite([
{
statement:
`UPDATE \`${TEST_TABLE}\` SET \`state\` = 'delivered' ` +
"WHERE `id` = ? AND `token` = ? AND `state` = 'sending'",
values: [id, token],
},
...entries,
]);
const [row] = await db().pread(
`SELECT \`state\`, \`token\` FROM \`${TEST_TABLE}\` WHERE \`id\` = ?`,
[id],
);
return row?.state === 'delivered' && row.token === token;
},
});
const write = (
sendKey: string,
tx: DeferredWriteTx,
over: {
payload?: string | null;
appUid?: string;
signal?: AbortSignal;
} = {},
) =>
driver().deferredAppMessages.write(
{
holderUserId: holder.user!.id,
appUid: over.appUid ?? app.uid,
payload:
over.payload === undefined
? JSON.stringify({ message: { title: 'Later' } })
: over.payload,
sendKey,
signal: over.signal ?? new AbortController().signal,
},
tx,
);
const fakeSchedules = (plan: Partial<DeferredSendPlan>) =>
({
onSubscriptionsChanged: vi.fn(async () => {}),
createFor: vi.fn(),
list: vi.fn(),
get: vi.fn(),
update: vi.fn(),
remove: vi.fn(),
receivePeerChanges: vi.fn(),
prepareSendAt: vi.fn(async (): Promise<DeferredSendPlan> => ({
sendAt: 0,
immediate: false,
holderUserId: holder.user!.id,
appUid: app.uid,
handlerDepth: 0,
quota: {
recurringPerUser: 1,
recurringPerApp: 1,
oneShotsPerUser: 1,
oneShotsPerApp: 1,
budgetPerUser: 1,
budgetPerApp: 1,
},
...plan,
})),
createDeferredAppMessage: vi.fn(async (p: DeferredSendPlan) => ({
scheduleId: 'sched-1',
sendAt: p.sendAt,
})),
}) satisfies ScheduleServiceContract;
beforeAll(async () => {
server = await setupTestServer({
events: { enabled: true },
} as IConfig);
const username = `sendat-${Math.random().toString(36).slice(2, 10)}`;
const user = await server.stores.user.create({
username,
uuid: uuidv4(),
password: null,
email: `${username}@test.local`,
requires_email_confirmation: false,
});
const row = (await server.stores.user.getById(user.id))!;
holder = makeActor({
user: {
id: row.id,
uuid: row.uuid,
username: row.username,
email: row.email ?? null,
email_confirmed: true,
} as Actor['user'],
});
const name = `sendat-app-${Math.random().toString(36).slice(2, 10)}`;
app = (await server.stores.app.create(
{ name, title: name, index_url: `https://${name}.test/` },
{ ownerUserId: row.id },
)) as { id: number; uid: string };
await server.services.permission.grantUserAppPermission(
holder,
app.uid,
EVENTS_BACKGROUND_PERMISSION,
);
await db().write(
`CREATE TABLE \`${TEST_TABLE}\` (\`id\` INTEGER PRIMARY KEY, ` +
'`token` VARCHAR(64) NOT NULL, `state` VARCHAR(16) NOT NULL)',
[],
);
saved = slot().schedules;
}, BOOT_TIMEOUT_MS);
beforeEach(async () => {
await db().write('DELETE FROM `notification` WHERE `app_uid` = ?', [
app.uid,
]);
});
afterEach(() => {
setSchedules(saved);
vi.restoreAllMocks();
});
afterAll(async () => {
await server?.shutdown();
});
describe('notify({ sendAt }) without services.schedules', () => {
it('is refused events_disabled, even when due now, storing nothing', async () => {
setSchedules(undefined);
expect(await codeOf(notify({ sendAt: Date.now() }))).toBe(
'events_disabled',
);
expect(await codeOf(notify({ sendAt: Date.now() + 3_600_000 }))).toBe(
'events_disabled',
);
expect(await mailbox()).toEqual([]);
});
it('sends as before without sendAt', async () => {
setSchedules(undefined);
const sent = (await notify({ text: 'now' })) as { uid: string };
expect(sent.uid).toBeTruthy();
expect(await mailbox()).toHaveLength(1);
});
});
describe('notify({ sendAt }) with services.schedules', () => {
it('hands a later send to createDeferredAppMessage and answers its id', async () => {
const at = Date.now() + 3_600_000;
const schedules = fakeSchedules({ sendAt: at });
setSchedules(schedules);
await expect(
notify({ sendAt: at, text: 'later', priority: 'high' }),
).resolves.toEqual({
scheduled: { scheduleId: 'sched-1', sendAt: at },
});
expect(schedules.prepareSendAt).toHaveBeenCalledWith(
expect.objectContaining({ user: holder.user }),
at,
);
expect(schedules.createDeferredAppMessage).toHaveBeenCalledWith(
expect.objectContaining({ sendAt: at }),
{
message: expect.objectContaining({
title: 'Hi',
text: 'later',
priority: 'high',
}),
},
);
expect(await mailbox()).toEqual([]);
});
it('sends at once when the plan says it is due', async () => {
const schedules = fakeSchedules({ immediate: true });
setSchedules(schedules);
const sent = (await notify({ sendAt: Date.now() })) as {
uid: string;
};
expect(sent.uid).toBeTruthy();
expect(schedules.createDeferredAppMessage).not.toHaveBeenCalled();
expect(await mailbox()).toHaveLength(1);
});
it('passes a refusal from prepareSendAt through', async () => {
const schedules = fakeSchedules({});
schedules.prepareSendAt.mockRejectedValueOnce(
Object.assign(new Error('scoped'), {}),
);
setSchedules(schedules);
await expect(notify({ sendAt: Date.now() })).rejects.toThrow('scoped');
expect(await mailbox()).toEqual([]);
});
});
describe('deferredAppMessages.write', () => {
it('writes one row keyed by the send, in the caller`s transaction', async () => {
const token = uuidv4();
const key = uuidv4();
const id = await claimed(token);
await expect(
write(key, txFor(id, token), {
payload: JSON.stringify({
message: { title: 'Later', priority: 'high' },
}),
}),
).resolves.toBe('delivered');
const rows = await mailbox();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ uid: key, send_key: key });
expect(valueOf(rows[0])).toMatchObject({
title: 'Later',
priority: 'high',
});
});
it('writes nothing when the guard no longer holds: another claim took it', async () => {
const id = await claimed(uuidv4());
const key = uuidv4();
await expect(write(key, txFor(id, uuidv4()))).resolves.toBe('lost');
expect(await mailbox()).toEqual([]);
});
it('replaces the open row of its group, which takes the key', async () => {
const open = (await notify({ group: 'g', text: 'first' })) as {
uid: string;
};
const token = uuidv4();
const key = uuidv4();
const id = await claimed(token);
await expect(
write(key, txFor(id, token), {
payload: JSON.stringify({
message: { title: 'Later', group: 'g' },
}),
}),
).resolves.toBe('delivered');
const rows = await mailbox();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ uid: open.uid, send_key: key });
expect(valueOf(rows[0])).toMatchObject({ title: 'Later' });
});
it('drops a payload it can`t read, and a sender whose app is gone', async () => {
const token = uuidv4();
const id = await claimed(token);
await expect(
write(uuidv4(), txFor(id, token), { payload: '{"nope":1}' }),
).resolves.toBe('dropped');
await expect(
write(uuidv4(), txFor(id, token), { appUid: `app-${uuidv4()}` }),
).resolves.toBe('dropped');
expect(await mailbox()).toEqual([]);
});
it('writes nothing once the caller`s claim is lost', async () => {
const token = uuidv4();
const id = await claimed(token);
const lost = new AbortController();
lost.abort();
await expect(
write(uuidv4(), txFor(id, token), { signal: lost.signal }),
).rejects.toThrow();
expect(await mailbox()).toEqual([]);
});
});
/**
* A second claim taking a send over while the first one's transaction is open.
* Needs two writers at once with row locks; sqlite serialises writers, so this
* runs on postgres only.
*/
describe.runIf(process.env.PUTER_TEST_DB_ENGINE === 'postgres')(
'a deferred write racing another writer',
() => {
/**
* Hold the first statement matching `match` on its connection, before
* or after it runs, until released.
*/
const holdStatement = (match: RegExp, when: 'before' | 'after') => {
const client = db() as unknown as {
query: (
target: unknown,
sql: string,
params?: unknown[],
) => Promise<unknown>;
};
const original = client.query.bind(client);
let reached!: () => void;
let release!: () => void;
const atBarrier = new Promise<void>((r) => (reached = r));
const released = new Promise<void>((r) => (release = r));
let held = false;
vi.spyOn(client as never, 'query' as never).mockImplementation(
(async (target: unknown, sql: string, params?: unknown[]) => {
if (held || !match.test(sql))
return original(target, sql, params);
held = true;
if (when === 'before') {
reached();
await released;
return original(target, sql, params);
}
const result = await original(target, sql, params);
reached();
await released;
return result;
}) as never,
);
return { atBarrier, release };
};
const DELIVERED = /SET `state` = 'delivered'/;
const REPLACE = /`replaced_at` = \?, `send_key` = \? WHERE/;
/** Whether `promise` is still pending after `ms`. */
const stillPending = async (promise: Promise<unknown>, ms: number) => {
let settled = false;
const watched = (async () => {
try {
await promise;
} finally {
settled = true;
}
})();
await new Promise((r) => setTimeout(r, ms));
void watched;
return !settled;
};
it('a transaction that opened before a takeover writes nothing', async () => {
const stale = uuidv4();
const id = await claimed(stale);
const key = uuidv4();
const gate = holdStatement(DELIVERED, 'before');
const first = write(key, txFor(id, stale));
await gate.atBarrier;
await db().write(
`UPDATE \`${TEST_TABLE}\` SET \`token\` = ? WHERE \`id\` = ?`,
[uuidv4(), id],
);
gate.release();
await expect(first).resolves.toBe('lost');
expect(
await server.stores.notification.getBySendKey(key),
).toBeNull();
expect(await mailbox()).toEqual([]);
});
it('an immediate replacement waiting on the open transaction lands after it, without the key', async () => {
const open = (await notify({ group: 'g', text: 'first' })) as {
uid: string;
};
const token = uuidv4();
const id = await claimed(token);
const key = uuidv4();
const gate = holdStatement(REPLACE, 'after');
const deferred = write(key, txFor(id, token), {
payload: JSON.stringify({
message: { title: 'Later', group: 'g', text: 'later' },
}),
});
await gate.atBarrier;
const immediate = notify({ group: 'g', text: 'newer' });
// Past the group lock's wait, then blocked on the row.
expect(await stillPending(immediate, 2_500)).toBe(true);
gate.release();
await expect(deferred).resolves.toBe('delivered');
await immediate;
const rows = await mailbox();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ uid: open.uid, send_key: null });
expect(valueOf(rows[0]).text).toBe('newer');
});
},
);
@@ -859,32 +859,3 @@ describe('NotificationDriver — rows written before the scope columns', () => {
expect(await selectUids(holder.actor)).toEqual([uid]);
});
});
describe('NotificationDriver displayed notification catch-up', () => {
it('selects named ids within the app and user scope', async () => {
const box = await mailbox();
await server.services.notification.markAcknowledged(box.rows.appUser, box.holder.userId);
const rows = await withActor(box.ownAppActor, () => driver.select({
subject: 'notif:app-user',
uids: Object.values(box.rows),
limit: 50,
}));
expect(rows).toEqual([expect.objectContaining({
uid: box.rows.appUser, acknowledged: expect.any(Number),
})]);
});
it.each([Array(51).fill('one'), [''], [7], 'one', ['x'.repeat(129)]])(
'rejects an invalid or oversized id batch: %s', async (uids) => {
const { actor } = await makeUser();
await expect(withActor(actor, () => driver.select({ uids })))
.rejects.toMatchObject({ statusCode: 400 });
},
);
it('returns no rows for an empty id batch', async () => {
const { actor } = await makeUser();
expect(await withActor(actor, () => driver.select({ uids: [] }))).toEqual([]);
});
});
@@ -37,7 +37,6 @@ import {
APP_MESSAGE_TYPE,
appMessageSender,
parseAppMessage,
type AppMessage,
} from '../../services/notification/appMessage.js';
import {
APP_READABLE_AUDIENCES,
@@ -46,20 +45,11 @@ import {
ownedAppUids,
} from '../../services/notification/notificationAudience.js';
import {
assertNotificationValueFits,
NOTIFICATION_VALUE_MAX_BYTES,
OUTSIDE_ACCOUNT_MAILBOX_TYPES,
resolveNotificationWrite,
type NotificationTypeName,
} from '../../services/notification/notificationTypes.js';
import {
schedulesUnavailable,
type DeferredAppMessage,
type DeferredSendPlan,
type ScheduleServiceContract,
type SqlGuard,
} from '../../core/schedules.js';
import type { BatchEntry } from '../../clients/database/DatabaseClient.js';
import { withRedisLock } from '../../util/redisLock.js';
import { PuterDriver } from '../types.js';
import type { Actor } from '../../core/actor.js';
import type { DriverRateLimitConfig } from '../meta.js';
@@ -68,50 +58,6 @@ import { CRUD_CONCURRENT } from '../util/crudLimits.js';
const MAX_SELECT_LIMIT = 200;
const MINUTE_MS = 60_000;
const DAY_MS = 24 * 60 * MINUTE_MS;
// A group's sends queue for up to ~1 s; a stalled holder blocks them for the TTL.
const GROUP_LOCK_MS = 5_000;
const GROUP_LOCK_ATTEMPTS = 40;
const GROUP_LOCK_RETRY_MS = 25;
type AppMessageWindow = 'minute' | 'day';
export type { SqlGuard };
/** What a delayed app message's fire-time write needs from its row. */
export interface DeferredWriteInput {
holderUserId: number;
appUid: string;
/** The stored {@link DeferredAppMessage}, as JSON text. */
payload: string | null;
/** The slot's key: the written row's uid or `send_key`, and its `sendId`. */
sendKey: string;
/** Aborted when the caller's claim is lost. */
signal: AbortSignal;
}
/**
* The caller's transaction the mailbox write commits in. Atomicity is the
* caller's to keep; the type can't enforce it.
*/
export interface DeferredWriteTx {
/**
* What every mailbox statement is fenced on: the caller's row marked
* delivered by this claim.
*/
guard: SqlGuard;
/**
* Must be the whole delivery in one step: the caller's guarded transition
* of its row to delivered and `entries`, in that order, in one
* `batchWrite`, then a read of the row on the primary. True only when that
* read shows this claim holding the row as delivered. Splitting the
* transition from `entries` lets one slot write two rows.
*/
commit(entries: readonly BatchEntry[]): Promise<boolean>;
}
/** How a delayed send's write stood when it stopped. */
export type DeferredWriteOutcome =
'delivered' | 'dropped' | 'lost' | 'senderSuspended';
/** The mailbox slice a call named, in the terms the query takes it. */
interface MailboxScope {
@@ -126,52 +72,6 @@ const typeNotAllowed = (message: string): HttpError =>
code: 'notification_type_not_allowed',
});
const tooManyAppMessages = (): HttpError =>
new HttpError(429, 'Too many notifications from this app.', {
legacyCode: 'too_many_requests',
});
/** One refusal for either limit a notification through a share can hit. */
const tooManySharedNotifications = (): HttpError =>
new HttpError(429, 'Too many notifications.', {
legacyCode: 'too_many_requests',
});
/** The handle in `to`, which must be `{ handle }` and nothing else. */
const parseNotifyTarget = (to: unknown): string => {
const target = to as Record<string, unknown> | null;
if (
!target ||
typeof target !== 'object' ||
Array.isArray(target) ||
Object.keys(target).length !== 1 ||
typeof target.handle !== 'string' ||
target.handle === ''
) {
throw new HttpError(
400,
'`to` must be an object with only a string `handle`',
{ legacyCode: 'bad_request', fields: { field: 'to' } },
);
}
return target.handle;
};
/** A stored delayed message, or `null` for one that can't be read back. */
const parseDelayedAppMessage = (
payload: string | null,
): DeferredAppMessage | null => {
try {
const stored = JSON.parse(payload ?? 'null') as DeferredAppMessage;
if (typeof stored?.message?.title !== 'string') return null;
if (stored.to !== undefined && typeof stored.to !== 'string')
return null;
return stored;
} catch {
return null;
}
};
/** Whether a row belongs to the slice, as the same SQL scope selected it. */
const inScope = (
row: Record<string, unknown>,
@@ -202,8 +102,8 @@ const inScope = (
* grant changes that. A slice an actor may not see comes back empty rather
* than refused — the mailbox is not an oracle.
* - The holder creates any registered type but `app.message`, silently. An actor
* holding an app creates `app.message` only, for its own user or, through a
* KV share handle, for the share's other party, and it is delivered live.
* holding an app creates `app.message` only, for its own user, and it is
* delivered live.
*
* `read` and `select` also take an optional `subject` naming one slice
* (`notif:<appUid>:<audience>`); the two-segment form expands from the actor's
@@ -255,7 +155,17 @@ export class NotificationDriver extends PuterDriver {
legacyCode: 'bad_request',
});
}
// Its own field checks come first, so an app hears which field is wrong.
if (
Buffer.byteLength(JSON.stringify(value), 'utf8') >
NOTIFICATION_VALUE_MAX_BYTES
) {
throw new HttpError(
400,
`\`value\` may not exceed ${NOTIFICATION_VALUE_MAX_BYTES} bytes`,
{ legacyCode: 'notification_value_too_large' },
);
}
if (actor.effectiveApp) {
return this.#createAppMessage(
actor,
@@ -264,7 +174,6 @@ export class NotificationDriver extends PuterDriver {
value,
);
}
assertNotificationValueFits(value);
if (type === APP_MESSAGE_TYPE) {
throw typeNotAllowed(`Only apps can create ${APP_MESSAGE_TYPE}`);
}
@@ -289,12 +198,11 @@ export class NotificationDriver extends PuterDriver {
// Silent: a driver `create` has never pushed to the live socket, only
// ever written the row — moving through the registry does not change
// that.
const uid = await this.#write(actor.user.id, value, {
return this.#write(actor, value, {
type: registeredType,
appUid,
silent: true,
});
return this.#readBack(uid, actor.user.id);
}
async read(args: Record<string, unknown>): Promise<unknown> {
@@ -329,34 +237,13 @@ export class NotificationDriver extends PuterDriver {
: predicate;
const scope = this.#resolveScope(actor, args.subject);
const uids = args.uids;
if (
uids !== undefined &&
(!Array.isArray(uids) ||
uids.length > 50 ||
uids.some(
(uid) =>
typeof uid !== 'string' ||
uid.length === 0 ||
uid.length > 128,
))
) {
throw new HttpError(
400,
'uids must contain at most 50 notification ids',
{
legacyCode: 'bad_request',
},
);
}
const query: {
limit: number;
scope: MailboxScope | null;
filter?: string;
onlyUnacknowledged?: boolean;
excludeTypes?: readonly string[];
uids?: string[];
} = { limit, scope, ...(uids ? { uids: uids as string[] } : {}) };
} = { limit, scope };
// The whole mailbox, or a slice naming no app, is the desktop's view of
// it, which leaves out the types only an app has a use for. A slice
// naming the app still returns them.
@@ -413,17 +300,14 @@ export class NotificationDriver extends PuterDriver {
return { success: !!ok };
}
/**
* An app's message to its own user, or with `to.handle` to the other party
* of a KV share. The actor names the app; the share names the recipient.
*/
/** An app's message to its own user; the actor names recipient and app. */
async #createAppMessage(
actor: UserActor,
appUid: string,
object: Record<string, unknown>,
value: Record<string, unknown>,
): Promise<unknown> {
const { type, to, sendAt, ...fields } = value;
const { type, ...fields } = value;
if (type !== APP_MESSAGE_TYPE) {
throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
}
@@ -436,343 +320,70 @@ export class NotificationDriver extends PuterDriver {
);
}
}
const handle = to === undefined ? undefined : parseNotifyTarget(to);
const message = parseAppMessage(fields);
assertNotificationValueFits(value);
const timing =
sendAt === undefined || sendAt === null
? null
: await this.#schedules().prepareSendAt(actor, sendAt);
if (timing && !timing.immediate)
return this.#scheduleAppMessage(actor, message, handle, timing);
if (handle === undefined) {
const uid = await this.#sendAppMessage(
actor.user.id,
appUid,
message,
);
return this.#readBack(uid, actor.user.id);
}
try {
const recipient =
await this.services.kvShare.resolveNotifyRecipient(
actor,
handle,
);
await this.#sendAppMessage(
recipient.recipientUserId,
recipient.appUid,
message,
);
} catch (err) {
// Which limit refused would say how much the recipient is sent.
if (err instanceof HttpError && err.statusCode === 429)
throw tooManySharedNotifications();
throw err;
}
// Even a uid would say whether the recipient's last one was replaced.
return {};
}
/**
* A message with a `group` replaces the newest one in that group unless it
* was shown or dismissed, which spends a minute slot but no day slot.
* Limits are keyed on the recipient, who need not be the actor.
*/
async #sendAppMessage(
recipientId: number,
appUid: string,
message: AppMessage,
): Promise<string> {
return this.#oneSendPerGroup(recipientId, appUid, message, () =>
this.#replaceOrWriteAppMessage(recipientId, appUid, message),
);
}
/**
* One send per group at a time, so a burst into an empty group writes one
* row that the rest replace. A lock that is busy past the wait or
* unreachable still sends: the race costs a second row and its day slot.
*/
#oneSendPerGroup<T>(
recipientId: number,
appUid: string,
message: AppMessage,
send: () => Promise<T>,
): Promise<T> {
if (!message.group) return send();
return withRedisLock(
this.clients.redis,
`notif:app-message:group:${recipientId}:${appUid}:${message.group}`,
send,
{
ttlMs: GROUP_LOCK_MS,
attempts: GROUP_LOCK_ATTEMPTS,
retryMs: GROUP_LOCK_RETRY_MS,
onUnavailable: 'run',
onBusy: send,
},
);
}
// -- Delayed sends -----------------------------------------------
#schedules(): ScheduleServiceContract {
const schedules = this.services.schedules;
if (!schedules) throw schedulesUnavailable();
return schedules;
}
/**
* Store an app's message to go out at `sendAt`. A recipient named through a
* share is checked now without spending its budget; every limit is spent
* when it goes, and the share is checked again then.
*/
async #scheduleAppMessage(
actor: UserActor,
message: AppMessage,
handle: string | undefined,
plan: DeferredSendPlan,
): Promise<{ scheduled: { scheduleId: string; sendAt: number } }> {
if (handle !== undefined)
await this.services.kvShare.resolveNotifyRecipient(actor, handle, {
spendBudget: false,
if (!(await this.#withinAppMessageLimits(actor.user.id, appUid))) {
throw new HttpError(429, 'Too many notifications from this app.', {
legacyCode: 'too_many_requests',
});
return {
scheduled: await this.#schedules().createDeferredAppMessage(
plan,
handle === undefined ? { message } : { message, to: handle },
),
};
}
/** For extensions. A field, not a method, so `/drivers/call` can't reach it. */
readonly deferredAppMessages = {
write: (input: DeferredWriteInput, tx: DeferredWriteTx) =>
this.#writeDeferredAppMessage(input, tx),
};
/**
* Check the sender and recipient as they stand now, spend the recipient's
* limits, and write the mailbox row through `tx`, fenced on its guard.
* Refusals are the same `HttpError`s an immediate send throws.
*/
async #writeDeferredAppMessage(
input: DeferredWriteInput,
tx: DeferredWriteTx,
): Promise<DeferredWriteOutcome> {
const { holderUserId, sendKey, signal } = input;
const sender = await this.services.events.resolveScheduleActor(
holderUserId,
input.appUid,
);
if (!sender) {
// A lagging replica can miss an app the primary has: retried.
const there =
(await this.stores.app.uidExistsOnPrimary(input.appUid)) &&
(await this.stores.user.getById(holderUserId)) !== null;
if (there) throw new Error('the sender could not be resolved');
return 'dropped';
}
if (sender.user?.suspended) return 'senderSuspended';
// Uninstalled since, and the cancel at uninstall missed it.
const appId = sender.effectiveApp?.id;
if (
typeof appId !== 'number' ||
!(await this.stores.permission.hasAnyUserAppPerm(
holderUserId,
appId,
))
)
return 'dropped';
const stored = parseDelayedAppMessage(input.payload);
if (!stored) return 'dropped';
const { message } = stored;
let recipientId = holderUserId;
let appUid = input.appUid;
if (stored.to !== undefined) {
const recipient =
await this.services.kvShare.resolveNotifyRecipient(
sender,
stored.to,
);
recipientId = recipient.recipientUserId;
appUid = recipient.appUid;
}
return this.#oneSendPerGroup(recipientId, appUid, message, async () => {
// Primary, as for an immediate send.
const open = message.group
? await this.stores.notification.getUnseenByGroup(
recipientId,
appUid,
message.group,
{ primary: true },
)
: null;
await this.#spendAppMessageLimits(
recipientId,
appUid,
open ? ['minute'] : ['minute', 'day'],
);
message.app = appMessageSender(
appUid,
await this.stores.app.getByUid(appUid),
);
// Another attempt may hold the lease now; hand the claim back.
if (signal.aborted) throw new Error('delayed send lost its lease');
const registered = resolveNotificationWrite(
APP_MESSAGE_TYPE,
appUid,
);
const delivered = await tx.commit(
this.stores.notification.sendEntries(
{
sendKey,
userId: recipientId,
value: { ...message, type: registered.type },
type: registered.type,
audience: registered.audience,
appUid,
group: message.group ?? null,
replaceUid: open ? String(open.uid) : null,
at: Date.now(),
},
tx.guard,
),
);
if (!delivered) return 'lost';
if (open)
await this.#spendDayIfFresh(recipientId, appUid, open, sendKey);
return 'delivered';
});
}
/**
* A replacement that found its row shown since the lookup wrote a new one,
* which costs a day slot. Already written, so a full day is not a refusal.
*/
async #spendDayIfFresh(
recipientId: number,
appUid: string,
open: Record<string, unknown>,
sendKey: string,
): Promise<void> {
try {
const written =
await this.stores.notification.getBySendKey(sendKey);
if (written && written.uid !== open.uid)
await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
} catch {
// The row is written; a slot not spent is the only loss.
}
}
async #replaceOrWriteAppMessage(
recipientId: number,
appUid: string,
message: AppMessage,
): Promise<string> {
// Primary: a lagging replica would turn a replacement into a new
// notification, which spends a day slot.
const open = message.group
? await this.stores.notification.getUnseenByGroup(
recipientId,
appUid,
message.group,
{ primary: true },
)
: null;
await this.#spendAppMessageLimits(
recipientId,
appUid,
open ? ['minute'] : ['minute', 'day'],
);
message.app = appMessageSender(
appUid,
await this.stores.app.getByUid(appUid),
);
if (open) {
const uid = String(open.uid);
if (
await this.services.notification.notifyUpdate(
uid,
recipientId,
message,
{ type: APP_MESSAGE_TYPE, replace: true },
)
) {
return uid;
}
// Shown or dismissed since the lookup, so this one is new after all.
await this.#spendAppMessageLimits(recipientId, appUid, ['day']);
}
return this.#write(recipientId, message, {
return this.#write(actor, message, {
type: APP_MESSAGE_TYPE,
appUid,
silent: false,
group: message.group ?? null,
});
}
/**
* Every window named is read before any is spent, so a refusal costs none.
* Both windows are read before either is spent, so a refusal costs none.
* The minute window is spent first: it is the one most likely to run out
* between the read and the spend, and a refusal there leaves the day
* alone.
*/
async #spendAppMessageLimits(
async #withinAppMessageLimits(
userId: number,
appUid: string,
spend: readonly AppMessageWindow[],
): Promise<void> {
): Promise<boolean> {
const configured = this.config.notificationAppMessageLimits ?? {};
const all: Record<AppMessageWindow, [string, number, number]> = {
minute: [
const all: Array<[string, number, number]> = [
[
`notif:app-message:minute:${userId}:${appUid}`,
configured.perMinute ?? APP_MESSAGE_RATE_LIMITS.perMinute,
MINUTE_MS,
],
day: [
[
`notif:app-message:day:${userId}:${appUid}`,
configured.perDay ?? APP_MESSAGE_RATE_LIMITS.perDay,
DAY_MS,
],
};
const windows = spend
.map((name) => all[name])
.filter(([, limit]) => limit > 0);
];
const windows = all.filter(([, limit]) => limit > 0);
for (const [key, limit, windowMs] of windows) {
if (!(await peekRateLimit(key, limit, windowMs))) {
throw tooManyAppMessages();
}
if (!(await peekRateLimit(key, limit, windowMs))) return false;
}
for (const [key, limit, windowMs] of windows) {
if (!(await checkRateLimit(key, limit, windowMs))) {
throw tooManyAppMessages();
}
if (!(await checkRateLimit(key, limit, windowMs))) return false;
}
return true;
}
/** Writes for `userId`, who need not be the actor, and names the row. */
async #write(
userId: number,
actor: UserActor,
value: Record<string, unknown>,
opts: {
type: NotificationTypeName;
appUid: string | null;
silent: boolean;
group?: string | null;
},
): Promise<string> {
): Promise<unknown> {
const uid = await this.services.notification.notify(
[userId],
[actor.user.id],
value,
opts,
);
@@ -781,17 +392,10 @@ export class NotificationDriver extends PuterDriver {
legacyCode: 'internal_error',
});
}
return uid;
}
/** The service hands back only the uid; the primary has the rest. */
async #readBack(
uid: string,
userId: number,
): Promise<Record<string, unknown> | null> {
// `notify` hands back only the uid; the primary has the rest.
return this.#toClient(
await this.stores.notification.getByUid(uid, {
userId,
userId: actor.user.id,
primary: true,
}),
);
@@ -129,27 +129,6 @@ describe('pipeStreamResult', () => {
expect(warn).not.toHaveBeenCalled();
});
it('adds at most two close listeners to the response', async () => {
// The response already carries close listeners from metering,
// compression and tracing; Node warns past ten.
let added = -1;
server = createServer((_req, res) => {
const before = res.listenerCount('close');
pipeStreamResult(Readable.from(['a', 'b']), res, 'test.run');
added = res.listenerCount('close') - before;
});
await new Promise<void>((resolve) => server!.listen(0, resolve));
const { port } = server.address() as AddressInfo;
await new Promise<void>((resolve) =>
get(`http://127.0.0.1:${port}/`, (res) => {
res.resume();
res.on('end', () => resolve());
}),
);
expect(added).toBeGreaterThan(0);
expect(added).toBeLessThanOrEqual(2);
});
it('drops the source, without throwing, when the client left before the result was ready', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const source = Readable.from(['late']);
+8 -16
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type { Readable } from 'node:stream';
import { pipeline, type Readable } from 'node:stream';
// Only what's used here, so callers on another @types/node (or express's
// Response) still fit.
@@ -32,32 +32,24 @@ type ResponseSink = NodeJS.WritableStream & {
* mid-body destroys the response, so the client sees an aborted transfer rather
* than a clean end, and the failure is logged once instead of surfacing as an
* unhandled stream error. A client that hangs up stops the source.
*
* `pipe` rather than `pipeline`: pipeline puts seven close listeners on the
* response, and with the ones metering, compression and tracing add, every
* streamed response tripped Node's ten-listener warning.
*/
export function pipeStreamResult(
stream: Readable,
res: ResponseSink,
label: string,
): void {
// The caller hung up while the driver ran: nothing will read the source.
// The caller hung up while the driver ran; `pipeline` throws on a closed
// destination, so drop the source instead.
if (res.closed || res.destroyed) {
stream.destroy();
return;
}
stream.once('error', (err) => {
stream.unpipe(res);
pipeline(stream, res, (err) => {
if (!err) return;
if (!res.destroyed) res.destroy();
// The client hanging up is not a failure of ours.
if ((err as { code?: string }).code === 'ERR_STREAM_PREMATURE_CLOSE')
return;
console.warn(`[drivers] ${label} stream failed mid-response:`, err);
});
// A response that closes before the source ended is a client hanging up.
res.once('close', () => {
if (stream.readableEnded) return;
stream.unpipe(res);
stream.destroy();
});
res.once('error', () => stream.destroy());
stream.pipe(res);
}
+4 -63
View File
@@ -17,49 +17,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/** The code units `\s` matches. */
const isSpace = (c: number): boolean =>
c === 0x20 ||
(c >= 0x09 && c <= 0x0d) ||
(c >= 0xa0 &&
(c === 0xa0 ||
c === 0x1680 ||
(c >= 0x2000 && c <= 0x200a) ||
c === 0x2028 ||
c === 0x2029 ||
c === 0x202f ||
c === 0x205f ||
c === 0x3000 ||
c === 0xfeff));
/** Whitespace runs in `text.slice(0, end)`, one less than `split(/\s+/)` gives. */
const countSpaceRuns = (text: string, end: number): number => {
let runs = 0;
let inSpace = false;
for (let i = 0; i < end; i++) {
const space = isSpace(text.charCodeAt(i));
if (space && !inSpace) runs++;
inSpace = space;
}
return runs;
};
/** How much text is left to scan for word boundaries, and what was seen. */
export interface WordScanBudget {
chars: number;
scannedChars: number;
scannedRuns: number;
}
/** About 5 ms of scanning. */
const WORD_SCAN_CHARS = 1024 * 1024;
export const createWordScanBudget = (): WordScanBudget => ({
chars: WORD_SCAN_CHARS,
scannedChars: 0,
scannedRuns: 0,
});
/**
* Tokens a run of text is worth.
*
@@ -71,27 +28,11 @@ export const createWordScanBudget = (): WordScanBudget => ({
* Shared by the chat credit gate, the chat unreported-stream backstop, and
* image-prompt pricing, so a calibration moves all of them at once.
*
* Words are counted over at most `budget.chars` characters, which this call
* spends; text past that is given the word density scanned so far. Pass one
* budget across a request's parts to bound the scan per request.
*
* @see https://help.openai.com/en/articles/4936856
*/
export const estimateTextTokens = (
text: string,
budget: WordScanBudget = createWordScanBudget(),
): number => {
export const estimateTextTokens = (text: string): number => {
if (!text) return 0;
const scanned = Math.min(text.length, budget.chars);
const runs = countSpaceRuns(text, scanned);
budget.chars -= scanned;
budget.scannedChars += scanned;
budget.scannedRuns += runs;
const unscannedRuns =
budget.scannedChars > 0
? ((text.length - scanned) * budget.scannedRuns) /
budget.scannedChars
: 0;
const words = 1 + runs + unscannedRuns;
return Math.floor((text.length / 4 + words * (4 / 3)) / 2);
return Math.floor(
(text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2,
);
};
@@ -238,8 +238,6 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
success: true,
errors: [],
url: `https://${name}.puter.work`,
app_uid: null,
app_name: null,
});
const [url, init] = putCalls()[0]!;
@@ -257,165 +255,6 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
expect(Number(row!.root_dir_id)).toBe(entry.id);
});
it('names the app a bound worker runs as, and lists it with the worker', async () => {
const { user, actor } = await makeUser();
const app = await server.stores.app.create(
{
name: `bound-${user.username}`,
title: `bound-${user.username}`,
index_url: `https://bound-${user.username}.example.com/`,
},
{ ownerUserId: user.id },
);
const path = `/${user.username}/bound.js`;
await writeSource(actor, user.id, path, 'export default {}');
const name = `bound-${user.username}`;
const result = await inCtx(actor, () =>
target.create({ appId: app.uid, workerName: name, filePath: path }),
);
expect(result).toMatchObject({
success: true,
app_uid: app.uid,
app_name: app.name,
});
const listed = (
(await inCtx(actor, () =>
target.getFilePaths({ workerName: name }),
)) as Array<{ app_uid: string | null; app_name: string | null }>
)[0];
expect(listed).toMatchObject({ app_uid: app.uid, app_name: app.name });
});
describe('redeployed under a different app', () => {
const seedBound = async () => {
const { user, actor } = await makeUser();
const makeApp = (label: string) =>
server.stores.app.create(
{
name: `${label}-${user.username}`,
title: `${label}-${user.username}`,
index_url: `https://${label}-${user.username}.example.com/`,
},
{ ownerUserId: user.id },
);
const first = await makeApp('first');
const second = await makeApp('second');
const path = `/${user.username}/rebind.js`;
await writeSource(actor, user.id, path, 'v1');
const name = `rebind-${user.username}`;
const deploy = (appId: string) =>
inCtx(actor, () =>
target.create({ appId, workerName: name, filePath: path }),
);
await deploy(first.uid);
const firstToken = deployedToken();
return {
user,
actor,
path,
name,
first,
second,
deploy,
firstToken,
};
};
const appOf = async (token: string) =>
(await server.services.auth.authenticate(token)).actor
?.effectiveApp?.uid;
const listed = async (actor: Actor, name: string) =>
(
(await inCtx(actor, () =>
target.getFilePaths({ workerName: name }),
)) as Array<{ app_uid: string | null; app_name: string | null }>
)[0];
it('moves the worker row to the app its new token acts as', async () => {
const { actor, name, second, deploy } = await seedBound();
const result = await deploy(second.uid);
expect(result).toMatchObject({ app_uid: second.uid });
expect(await listed(actor, name)).toMatchObject({
app_uid: second.uid,
app_name: second.name,
});
const row = await server.stores.subdomain.getBySubdomain(
`workers.puter.${name}`,
);
expect(Number(row!.app_owner)).toBe(second.id);
});
it('hot-reloads as the new app, not the one it was first bound to', async () => {
const { user, actor, path, second, deploy } = await seedBound();
await deploy(second.uid);
const notifySpy = vi.spyOn(server.services.notification, 'notify');
await writeSource(actor, user.id, path, 'v2');
await waitFor(
() => notifySpy.mock.calls.length > 0,
'hot-reload notification',
);
expect(notifySpy).toHaveBeenCalledWith(
[user.id],
expect.anything(),
expect.objectContaining({ appUid: second.uid }),
);
});
it('drops the binding when redeployed with no app', async () => {
const { actor, name, deploy, firstToken } = await seedBound();
expect(await deploy('')).toMatchObject({ app_uid: null });
expect(await listed(actor, name)).toMatchObject({
app_uid: null,
app_name: null,
});
expect(await server.services.auth.authenticate(firstToken)).toEqual(
{ invalid: true },
);
expect(
(await server.services.auth.authenticate(deployedToken()))
.actor,
).toBeTruthy();
});
it('revokes the old app`s token once the new one is deployed', async () => {
const { second, deploy, firstToken } = await seedBound();
await deploy(second.uid);
expect(await server.services.auth.authenticate(firstToken)).toEqual(
{ invalid: true },
);
expect(await appOf(deployedToken())).toBe(second.uid);
});
it('leaves the old binding and its token alone when the redeploy fails', async () => {
const { name, first, second, deploy, firstToken } =
await seedBound();
fetchSpy.mockResolvedValue(
edgeResponse({
success: false,
errors: [{ message: 'script too large' }],
}),
);
expect(await deploy(second.uid)).toMatchObject({ success: false });
expect(await appOf(firstToken)).toBe(first.uid);
const row = await server.stores.subdomain.getBySubdomain(
`workers.puter.${name}`,
);
expect(Number(row!.app_owner)).toBe(first.id);
});
});
it('sends the puter_auth secret and puter_endpoint binding in the deploy metadata', async () => {
const { user, actor } = await makeUser();
const path = `/${user.username}/worker.js`;
@@ -754,50 +593,6 @@ describe('WorkerDriver.create with a configured deploy backend', () => {
expect(putCalls()).toHaveLength(1);
});
const uploadedMetadata = () => {
const [, init] = putCalls().at(-1)!;
const form = (init as RequestInit).body as FormData;
return JSON.parse(form.get('metadata') as string);
};
it('sends limits.cpu_ms in the upload metadata when cpuMs is set', async () => {
const { user, actor } = await makeUser();
const path = `/${user.username}/cpu.worker.js`;
await writeSource(actor, user.id, path, 'export default {}');
const scriptName = `dt-cpu-${user.username}`;
await inCtx(actor, () =>
target.create({
appId: '',
workerName: scriptName,
filePath: path,
[INTERNAL_DEPLOY_TARGET]: { scriptName, cpuMs: 50 },
}),
);
expect(uploadedMetadata().limits).toEqual({ cpu_ms: 50 });
});
it('sends no limits key without cpuMs, and none on an ordinary create', async () => {
const { user, actor } = await makeUser();
const path = `/${user.username}/nocpu.worker.js`;
await writeSource(actor, user.id, path, 'export default {}');
const scriptName = `dt-nocpu-${user.username}`;
await deployTo(actor, path, scriptName);
expect(uploadedMetadata()).not.toHaveProperty('limits');
const name = `plain-${user.username}`;
await inCtx(actor, () =>
target.create({ appId: '', workerName: name, filePath: path }),
);
const metadata = uploadedMetadata();
expect(metadata).not.toHaveProperty('limits');
expect(Object.keys(metadata)).toEqual([
'body_part',
'compatibility_flags',
'compatibility_date',
'bindings',
]);
});
it('refuses a file another user owns, even with manage on it', async () => {
const owner = await makeUser();
const deployer = await makeUser();
@@ -1524,7 +1319,6 @@ describe('WorkerDriver.getFilePaths source resolution', () => {
file_path: string | null;
file_uid: string | null;
app_uid: string | null;
app_name: string | null;
created_at: string | null;
}>;
@@ -1534,7 +1328,6 @@ describe('WorkerDriver.getFilePaths source resolution', () => {
expect(row.file_path).toBe(path);
expect(row.file_uid).toBe(entry.uuid);
expect(row.app_uid).toBeNull();
expect(row.app_name).toBeNull();
expect(row.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T/);
});
@@ -3,7 +3,6 @@
// however the interactions with the Puter API are
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
@@ -14,7 +13,6 @@ import {
import { Actor, makeActor as resolveActor } from '../../core/actor.js';
import { runWithContext } from '../../core/context.js';
import { PuterServer } from '../../server.js';
import { DEFAULT_FREE_SUBSCRIPTION } from '../../services/metering/consts.js';
import {
setupPuterTestEnv,
setupTestServer,
@@ -511,7 +509,7 @@ describe('WorkerDriver', () => {
);
});
it('returns app_uid and app_name for sandboxed/app-bound workers and null for user-scoped workers', async () => {
it('returns app_uid for sandboxed/app-bound workers and null for user-scoped workers', async () => {
const tag = `wkuid${Math.random().toString(36).slice(2, 8)}`;
const owner = await makeUser(tag);
const app = await server.stores.app.create(
@@ -540,21 +538,15 @@ describe('WorkerDriver', () => {
const seen = (await inCtx(
() => target.getFilePaths({}),
actorFor(owner),
)) as Array<{
name: string;
app_uid: string | null;
app_name: string | null;
}>;
)) as Array<{ name: string; app_uid: string | null }>;
const bound = seen.find((r) => r.name === `${tag}-bound`);
const plain = seen.find((r) => r.name === `${tag}-plain`);
expect(bound).toBeDefined();
expect(bound?.app_uid).toBe(app.uid);
expect(bound?.app_name).toBe(`app-${tag}`);
expect(plain).toBeDefined();
expect(plain?.app_uid).toBeNull();
expect(plain?.app_name).toBeNull();
});
});
@@ -621,7 +613,6 @@ describe('WorkerDriver', () => {
file_path: dir.path,
file_uid: dir.uuid,
app_uid: app.uid,
app_name: app.name,
created_at: expect.any(String),
},
]);
@@ -799,174 +790,6 @@ describe('WorkerDriver', () => {
});
});
describe('rented-unit cap', () => {
const FREE_CAP = 16;
const PAID_CAP = 100;
// The cap sits after the deploy-credentials gate, so the driver needs
// credentials to reach it; a create under the cap then fails on the
// missing source file, which is what tells it past the cap check.
let originalWorkersConfig: unknown;
let originalEvents: unknown;
beforeAll(() => {
const driverConfig = (
target as unknown as { config: Record<string, unknown> }
).config;
originalEvents = driverConfig.events;
driverConfig.events = { userHandlers: true };
originalWorkersConfig = driverConfig.workers;
driverConfig.workers = {
XAUTHKEY: 'test-key',
ACCOUNTID: 'test-account',
};
});
afterAll(() => {
const driverConfig = (
target as unknown as { config: Record<string, unknown> }
).config;
driverConfig.workers = originalWorkersConfig;
driverConfig.events = originalEvents;
});
const holding = (
plan: string,
held: { workers?: number; scopes?: number; owned?: number },
) => {
vi.spyOn(
server.services.metering,
'getActorSubscription',
).mockResolvedValue({ id: plan } as never);
vi.spyOn(
server.stores.subdomain,
'countByUserIdAndPrefix',
).mockResolvedValue(held.workers ?? 0);
vi.spyOn(
server.stores.eventHandler,
'countUserScopesWithHandlersForUser',
).mockResolvedValue(held.scopes ?? 0);
vi.spyOn(
server.stores.eventHandler,
'countAppsWithHandlersForOwner',
).mockResolvedValue(held.owned ?? 0);
};
afterEach(() => {
vi.restoreAllMocks();
});
const createWorker = () =>
inCtx(() =>
target.create({
appId: 'test-app',
workerName: 'capped',
filePath: '/test.js',
}),
);
it('keeps the flat 100-worker cap over ordinary workers with user scopes off', async () => {
const driverConfig = (
target as unknown as { config: Record<string, unknown> }
).config;
driverConfig.events = { userHandlers: false };
try {
// 16 workers is fine here, and scopes and owned apps don't count.
holding(DEFAULT_FREE_SUBSCRIPTION, {
workers: 16,
scopes: 50,
owned: 50,
});
const list = vi
.spyOn(server.stores.subdomain, 'listByUserIdAndPrefix')
.mockResolvedValue(
Array.from({ length: 16 }, () => ({})) as never,
);
await expect(createWorker()).rejects.not.toMatchObject({
statusCode: 403,
});
list.mockResolvedValue(
Array.from({ length: 100 }, () => ({})) as never,
);
await expect(createWorker()).rejects.toMatchObject({
statusCode: 403,
message: 'Worker limit reached (max 100)',
legacyCode: 'forbidden',
});
} finally {
driverConfig.events = { userHandlers: true };
}
});
it('refuses a free user one unit past the cap, across all three sources', async () => {
for (const held of [
{ workers: FREE_CAP },
{ scopes: FREE_CAP },
{ owned: FREE_CAP },
{ workers: 10, scopes: 4, owned: 2 },
]) {
holding(DEFAULT_FREE_SUBSCRIPTION, held);
await expect(createWorker()).rejects.toMatchObject({
statusCode: 403,
legacyCode: 'forbidden',
});
}
});
it('lets a free user create up to the cap', async () => {
holding(DEFAULT_FREE_SUBSCRIPTION, {
workers: 10,
scopes: 3,
owned: 2,
});
await expect(createWorker()).rejects.not.toMatchObject({
statusCode: 403,
});
});
it('refuses a paid user the unit after 100', async () => {
holding('paid-plan', { workers: 90, scopes: 5, owned: 5 });
await expect(createWorker()).rejects.toMatchObject({
statusCode: 403,
});
holding('paid-plan', { workers: 90, scopes: 5, owned: 4 });
await expect(createWorker()).rejects.not.toMatchObject({
statusCode: 403,
});
expect(PAID_CAP).toBe(100);
});
it('keeps a free user over the cap, redeploys what they hold, and refuses a new one', async () => {
holding(DEFAULT_FREE_SUBSCRIPTION, { workers: 20 });
const user = actor.user;
const exists = vi
.spyOn(server.stores.subdomain, 'getBySubdomain')
.mockImplementation(
async (name: string) =>
(name === 'workers.puter.capped'
? { user_id: user.id, uuid: 'existing-uuid' }
: null) as never,
);
const noApp = makeActor({ app: undefined });
const create = (workerName: string) =>
inCtx(
() =>
target.create({
appId: '',
workerName,
filePath: '/test.js',
}),
noApp,
);
await expect(create('capped')).rejects.not.toMatchObject({
statusCode: 403,
});
await expect(create('another')).rejects.toMatchObject({
statusCode: 403,
});
exists.mockRestore();
});
});
describe('admission bypass', () => {
const unverified = (): Actor =>
makeActor({
+38 -101
View File
@@ -43,7 +43,6 @@ import { PuterDriver } from '../types.js';
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
import { awaitLater } from '../../util/concurrency.js';
import { isUniqueViolation } from '../../util/dbError.js';
import { assertRentedUnitRoom } from '../../util/rentedUnits.js';
import { loadFileInput } from '../util/fileInput.js';
import {
normalizeLimit,
@@ -58,7 +57,6 @@ const CF_BASE_URL = 'https://api.cloudflare.com/client/v4/accounts';
export const WORKER_COMPATIBILITY_DATE = '2025-07-15';
export const WORKER_COMPATIBILITY_FLAGS = ['global_fetch_strictly_public'];
const WORKER_NAME_REGEX = /^[a-zA-Z0-9_-]+$/;
/** Cap on ordinary workers while user-scope handlers are off. */
const MAX_WORKERS_PER_USER = 100;
/**
@@ -181,17 +179,12 @@ export interface InternalDeployTarget {
* scripts belong to what.
*/
tags?: string[];
/**
* Per-invocation CPU limit in milliseconds, sent with the upload. Unset
* sends no limit. Ignored by the local runtime.
*/
cpuMs?: number;
}
const MAX_SOURCE_SIZE = 10 * 1024 * 1024; // 10 MB
// How far to scan an app's child apps when resolving which workers it may
// see. A user is capped at their plan's rented units, so only that many
// child apps can actually own a worker; the ceiling is generous slack over that.
// see. A user is capped at MAX_WORKERS_PER_USER workers, so only that many
// child apps can actually own one; the ceiling is generous slack over that.
const CHILD_APP_SCAN_LIMIT = 1000;
// -- Preamble --------------------------------------------------------
@@ -459,34 +452,26 @@ export class WorkerDriver extends PuterDriver {
this.#requireCfConfig();
// With user scopes on, a new worker is one more rented unit under the
// plan's cap, and redeploying an existing worker is never refused.
// Deploy targets create no worker row and are bounded by their caller.
if (this.config.events?.userHandlers === true) {
if (!skipAdmission && !deployTarget && !existingSub) {
await assertRentedUnitRoom(
{
stores: this.stores,
metering: this.services.metering,
},
actor,
);
}
} else {
const existingWorkers =
skipAdmission || deployTarget
? []
: await this.stores.subdomain.listByUserIdAndPrefix(
actor.user.id,
WORKER_SUBDOMAIN_PREFIX,
);
if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
throw new HttpError(
403,
`Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
{ legacyCode: 'forbidden' },
);
}
// Quota check — count existing workers.puter.* subdomains owned by user
//
// A deploy target creates no such row, so this count can neither see
// its scripts nor be grown by them; running it would only make an
// unrelated worker quota decide whether these deploy. Whatever bounds
// them has to be counted where they actually live, and belongs to the
// caller that owns their lifecycle.
const existingWorkers =
skipAdmission || deployTarget
? []
: await this.stores.subdomain.listByUserIdAndPrefix(
actor.user.id,
WORKER_SUBDOMAIN_PREFIX,
);
if (existingWorkers.length >= MAX_WORKERS_PER_USER) {
throw new HttpError(
403,
`Worker limit reached (max ${MAX_WORKERS_PER_USER})`,
{ legacyCode: 'forbidden' },
);
}
// Read source file. loadFileInput runs the read-ACL check internally
@@ -650,18 +635,7 @@ export class WorkerDriver extends PuterDriver {
runtimePreamble.source + sourceCode,
deployTarget,
);
if (existingSub && cfResult.success === true)
await this.#rebindAfterDeploy(existingSub, actor, workerName, {
appOwner: appOwnerId ?? null,
appUid: boundApp?.uid ?? null,
});
// The app the worker's token acts as: its KV, AppData, notifications
// and pushes are that app's.
return {
...cfResult,
app_uid: boundApp?.uid ?? null,
app_name: boundApp?.name ?? null,
};
return cfResult;
}
/** Tear down the caller's worker; a refused edge delete is a 502. */
@@ -726,35 +700,6 @@ export class WorkerDriver extends PuterDriver {
};
}
/**
* A redeploy under a different app (or none) moves the row to the app the
* new token acts as, so the listing, hot-reload and `create()` agree, and
* revokes the session the old binding ran on. Only once the new script is
* live: a failed redeploy leaves the old one running on its old token.
*/
async #rebindAfterDeploy(
row: SubdomainRow,
actor: Actor & { user: { id: number } },
workerName: string,
next: { appOwner: number | null; appUid: string | null },
): Promise<void> {
const current =
row.app_owner === null || row.app_owner === undefined
? null
: Number(row.app_owner);
if (current === next.appOwner) return;
await this.stores.subdomain.setAppOwner(
String(row.uuid),
next.appOwner,
{ userId: actor.user.id },
);
await this.services.auth.deleteWorkerSessionsByName(
workerName,
actor.user.id,
{ exceptAppUid: next.appUid },
);
}
/** Script, sessions, then row: a failed step leaves the row for a retry. */
async #teardown(
row: SubdomainRow,
@@ -882,17 +827,17 @@ export class WorkerDriver extends PuterDriver {
file_path = loaded?.path;
file_uid = loaded?.uuid;
}
const loadedApp =
typeof r.app_owner === 'number'
? appsById.get(r.app_owner)
: undefined;
let app_uid = null;
if (typeof r.app_owner === 'number') {
const loadedApp = appsById.get(r.app_owner);
app_uid = loadedApp?.uid ?? null;
}
return {
name,
url: `https://${name}.puter.work`,
file_path,
file_uid,
app_uid: loadedApp?.uid ?? null,
app_name: loadedApp?.name ?? null,
app_uid,
created_at: r.ts
? new Date(r.ts as string).toISOString()
: null,
@@ -955,7 +900,6 @@ export class WorkerDriver extends PuterDriver {
compatibility_flags: WORKER_COMPATIBILITY_FLAGS,
compatibility_date: WORKER_COMPATIBILITY_DATE,
...(target?.tags ? { tags: target.tags } : {}),
...(target?.cpuMs ? { limits: { cpu_ms: target.cpuMs } } : {}),
bindings: [
// A script deployed without a token gets no binding at all,
// rather than an empty one for its code to find.
@@ -1203,18 +1147,15 @@ export class WorkerDriver extends PuterDriver {
async #resolveWorkerAppBinding(
actor: Actor & { user: { id: number } },
requestedAppUid?: string,
): Promise<{ uid: string; id?: number; name: string | null } | null> {
): Promise<{ uid: string; id?: number } | null> {
// Self-binding, either implicit or named. Not every actor shape carries
// `app.id`, and none carries the name, so both come from the row.
// `app.id`, so fall back to a lookup rather than leaving the subdomain
// row unowned.
const ownApp = actor.effectiveApp;
const selfBinding = async () => {
const row = await this.stores.app.getByUid(ownApp!.uid);
return {
uid: ownApp!.uid,
id: ownApp!.id ?? row?.id,
name: row?.name ?? null,
};
};
const selfBinding = async () => ({
uid: ownApp!.uid,
id: ownApp!.id ?? (await this.stores.app.getByUid(ownApp!.uid))?.id,
});
if (!requestedAppUid) {
return ownApp ? await selfBinding() : null;
}
@@ -1227,11 +1168,7 @@ export class WorkerDriver extends PuterDriver {
// Root session: unchanged: bind to whatever it names. An unknown
// uid still mints a token, as it did before, and simply leaves the
// subdomain row unowned by any app.
return {
uid: requestedAppUid,
id: app?.id,
name: app?.name ?? null,
};
return { uid: requestedAppUid, id: app?.id };
}
if (!app || !(await this.#appIsOwnedByActorApp(app.id, actor))) {
@@ -1239,7 +1176,7 @@ export class WorkerDriver extends PuterDriver {
legacyCode: 'forbidden',
});
}
return { uid: app.uid, id: app.id, name: app.name ?? null };
return { uid: app.uid, id: app.id };
}
/**
+3 -5
View File
@@ -53,11 +53,11 @@
"mime-types": "^2.1.35",
"murmurhash": "^2.0.1",
"mysql2": "^3.24.6",
"nodemailer": "^10.1.4",
"nodemailer": "^9.1.1",
"openai": "^6.34.0",
"otpauth": "^9.5.2",
"pg": "^8.21.0",
"postal-mime": "^4.2.0",
"postal-mime": "^3.0.1",
"replicate": "^1.0.0",
"sharp": "^0.35.5",
"smtp-server": "^3.19.24",
@@ -67,8 +67,7 @@
"ua-parser-js": "^1.0.41",
"undici": "^7.30.1",
"uuid": "^14.0.3",
"validator": "^13.15.35",
"ws": "^8.22.0"
"validator": "^13.15.35"
},
"devDependencies": {
"@types/bcrypt": "^6.0.0",
@@ -82,7 +81,6 @@
"@types/pg": "^8.6.1",
"@types/smtp-server": "^3.5.13",
"@types/validator": "^13.15.10",
"@types/ws": "^8.18.2",
"pgmock": "^1.0.3",
"typescript": "^5.9.3",
"vite": "^8.1.6",
+20 -320
View File
@@ -36,15 +36,7 @@ import { Controller, Get } from './core/http/decorators.ts';
import { HttpError } from './core/http/HttpError.ts';
import { extension, extensionStore } from './extensions.ts';
import { PuterServer } from './server.ts';
import {
JobScheduler,
type JobDefinition,
} from './services/jobs/JobScheduler.ts';
import {
allocateEphemeralPort,
createTestUser,
setupTestServer,
} from './testUtil.ts';
import { allocateEphemeralPort, setupTestServer } from './testUtil.ts';
import type { IConfig } from './types';
/**
@@ -531,7 +523,6 @@ describe('PuterServer host header validation — custom domains with every host
describe('PuterServer request parsing', () => {
let server: PuterServer;
let port: number;
let token: string;
const routeCount = { before: 0 };
beforeAll(async () => {
@@ -560,10 +551,6 @@ describe('PuterServer request parsing', () => {
} as unknown as IConfig,
{ listen: true },
);
({ token } = await createTestUser(server, {
username: 'large_body_user',
password: 'large-body-password',
}));
});
afterAll(async () => {
@@ -573,82 +560,31 @@ describe('PuterServer request parsing', () => {
const host = () => `puter.localhost:${port}`;
const MiB = 1024 * 1024;
const postJson = (
path: string,
payload: string,
headers: Record<string, string> = {},
) =>
rawRequest(
port,
path,
{
host: `api.puter.localhost:${port}`,
'content-type': 'application/json',
'content-length': String(Buffer.byteLength(payload)),
...headers,
},
'POST',
payload,
);
it.each([
'/drivers/call',
'/puterai/openai/v1/chat/completions',
'/puterai/anthropic/v1/messages',
'/puterai/anthropic/v1/messages/count_tokens',
])(
'keeps the default JSON limit on %s without a credential',
async (path) => {
const payload = JSON.stringify({ text: 'x'.repeat(51 * MiB) });
expect((await postJson(path, payload)).status).toBe(413);
},
);
it('parses a chat call above 50 MiB for a signed-in caller', async () => {
const payload = JSON.stringify({
interface: 'puter-chat-completion',
method: 'no-such-method',
pad: 'x'.repeat(51 * MiB),
});
for (const contentType of [
'application/json',
'text/plain;actually=json',
]) {
const res = await postJson('/drivers/call', payload, {
])('accepts JSON above 50 MiB on %s before applying auth gates', async (path) => {
const payload = JSON.stringify({ text: 'x'.repeat(51 * 1024 * 1024) });
for (const contentType of ['application/json', 'text/plain;actually=json']) {
const res = await rawRequest(port, path, {
host: `api.puter.localhost:${port}`,
'content-type': contentType,
authorization: `Bearer ${token}`,
});
// The controller read `method` out of the body.
expect(res.status, contentType).toBe(404);
expect(res.body).toContain('no-such-method');
}
});
it('keeps the default JSON limit for other drivers', async () => {
const payload = JSON.stringify({
interface: 'puter-kvstore',
method: 'get',
pad: 'x'.repeat(51 * MiB),
});
// Routing matches these spellings too.
for (const path of ['/drivers/call', '/Drivers/Call/']) {
const res = await postJson(path, payload, {
authorization: `Bearer ${token}`,
});
expect(res.status, path).toBe(413);
}, 'POST', payload);
expect(res.status).toBe(401);
}
});
it.each([
['/drivers/call', 128],
['/puterai/anthropic/v1/messages', 128],
['/raw-probe', 50],
])('rejects bodies above the JSON limit on %s', async (path, limitMiB) => {
const payload = JSON.stringify({ text: 'x'.repeat(limitMiB * MiB) });
const res = await postJson(path, payload, {
authorization: `Bearer ${token}`,
});
'/puterai/anthropic/v1/messages',
'/puterai/anthropic/v1/messages/count_tokens',
'/raw-probe',
])('rejects bodies above the JSON limit on %s', async (path) => {
const limitMiB = path === '/raw-probe' ? 50 : 128;
const payload = JSON.stringify({ text: 'x'.repeat(limitMiB * 1024 * 1024) });
const res = await rawRequest(port, path, {
host: `api.puter.localhost:${port}`,
'content-type': 'application/json',
}, 'POST', payload);
expect(res.status).toBe(413);
});
@@ -1492,105 +1428,7 @@ describe('PuterServer keep-alive timeout', () => {
});
describe('PuterServer lifecycle hooks', () => {
it('fails startup before running later hooks when a start hook rejects', async () => {
const failure = new Error('startup failed');
const later = vi.fn();
class FailingStart extends JobScheduler {
override async onServerStart() {
throw failure;
}
}
class LaterStart extends JobScheduler {
override async onServerStart() {
later();
}
}
extension.registerService('failingStart', FailingStart as never);
extension.registerService('laterStart', LaterStart as never);
try {
await expect(setupTestServer()).rejects.toBe(failure);
expect(later).not.toHaveBeenCalled();
} finally {
delete (extensionStore.services as Record<string, unknown>)
.failingStart;
delete (extensionStore.services as Record<string, unknown>)
.laterStart;
}
});
it.each([false, true])(
'continues teardown after failing hooks with listener callback %s',
async (prepareFirst) => {
const port = await allocateEphemeralPort();
const server = await setupTestServer(
{ port } as unknown as IConfig,
{ listen: true },
);
const logged = vi
.spyOn(console, 'error')
.mockImplementation(() => {});
const ran: string[] = [];
class FailingHooks {
onServerPrepareShutdown() {
ran.push('prepare:failed');
throw new Error('prepare failed');
}
async onServerShutdown() {
ran.push('shutdown:failed');
throw new Error('shutdown failed');
}
}
(server.services as Record<string, unknown>).fails =
new FailingHooks();
(server.services as Record<string, unknown>).after = {
onServerPrepareShutdown: () => ran.push('prepare:after'),
onServerShutdown: () => ran.push('shutdown:after'),
};
(server.clients as Record<string, unknown>).last = {
onServerShutdown: () => ran.push('shutdown:client'),
};
try {
if (prepareFirst) {
await server.prepareShutdown();
await vi.waitFor(() =>
expect(ran).toContain('prepare:after'),
);
}
await server.shutdown();
expect(ran).toEqual([
'prepare:failed',
'prepare:after',
'shutdown:failed',
'shutdown:after',
'shutdown:client',
]);
expect(logged).toHaveBeenCalledWith(
'[server] onServerPrepareShutdown failed in FailingHooks',
expect.any(Error),
);
expect(logged).toHaveBeenCalledWith(
'[server] onServerShutdown failed in FailingHooks',
expect.any(Error),
);
await expect(
new Promise<void>((resolve, reject) => {
const socket = net.connect(port, '127.0.0.1');
socket.on('connect', () => {
socket.destroy();
resolve();
});
socket.on('error', reject);
}),
).rejects.toMatchObject({ code: 'ECONNREFUSED' });
} finally {
delete (server.services as Record<string, unknown>).fails;
await server.shutdown();
logged.mockRestore();
}
},
);
it('runs drain-start and prepare-shutdown bottom-up, once, before tearing down', async () => {
it('runs prepare-shutdown bottom-up, once, before tearing down', async () => {
const port = await allocateEphemeralPort();
const server = await setupTestServer({ port } as unknown as IConfig, {
listen: true,
@@ -1598,9 +1436,6 @@ describe('PuterServer lifecycle hooks', () => {
const order: string[] = [];
const probe = (label: string) => ({
onServerDrainStart: (info?: { drainMs?: number }) => {
order.push(`drain:${label}:${info?.drainMs}`);
},
onServerPrepareShutdown: () => {
order.push(`prepare:${label}`);
},
@@ -1618,15 +1453,10 @@ describe('PuterServer lifecycle hooks', () => {
(layer as Record<string, unknown>).hookOrderProbe = probe(label);
}
await server.prepareShutdown({ drainMs: 1_000 });
await server.prepareShutdown();
await server.shutdown();
expect(order).toEqual([
'drain:client:1000',
'drain:store:1000',
'drain:service:1000',
'drain:controller:1000',
'drain:driver:1000',
'prepare:client',
'prepare:store',
'prepare:service',
@@ -1639,136 +1469,6 @@ describe('PuterServer lifecycle hooks', () => {
'shutdown:client',
]);
});
it('keeps accepting connections until every drain-start hook settles, without one holding up another', async () => {
const port = await allocateEphemeralPort();
const server = await setupTestServer({ port } as unknown as IConfig, {
listen: true,
});
let release!: () => void;
const held = new Promise<void>((resolve) => (release = resolve));
let otherRan = false;
(server.services as unknown as Record<string, unknown>).holdsListener =
{ onServerDrainStart: () => held };
(server.drivers as unknown as Record<string, unknown>).other = {
onServerDrainStart: () => {
otherRan = true;
},
};
const connect = () =>
new Promise<void>((resolve, reject) => {
const socket = net.connect(port, '127.0.0.1');
socket.on('connect', () => {
socket.destroy();
resolve();
});
socket.on('error', reject);
});
const prepared = server.prepareShutdown({ drainMs: 1_000 });
await new Promise((resolve) => setTimeout(resolve, 20));
expect(otherRan).toBe(true);
await expect(connect()).resolves.toBeUndefined();
release();
await prepared;
await expect(connect()).rejects.toMatchObject({
code: 'ECONNREFUSED',
});
await server.shutdown();
});
it('runs every drain-start hook when one of them throws', async () => {
const port = await allocateEphemeralPort();
const server = await setupTestServer({ port } as unknown as IConfig, {
listen: true,
});
vi.spyOn(console, 'error').mockImplementation(() => {});
const ran: string[] = [];
(server.stores as unknown as Record<string, unknown>).fails = {
onServerDrainStart: () => {
throw new Error('drain hook failed');
},
};
(server.services as unknown as Record<string, unknown>).after = {
onServerDrainStart: () => {
ran.push('after');
},
};
await server.prepareShutdown({ drainMs: 1_000 });
await server.shutdown();
expect(ran).toEqual(['after']);
expect(console.error).toHaveBeenCalledWith(
expect.stringContaining('onServerDrainStart failed'),
expect.any(Error),
);
});
});
describe('PuterServer shutdown during prepare', () => {
it('waits for prepare hooks the drain already started before tearing down', async () => {
const port = await allocateEphemeralPort();
const server = await setupTestServer({ port } as unknown as IConfig, {
listen: true,
});
const order: string[] = [];
(server.services as unknown as Record<string, unknown>).slowPrepare = {
onServerPrepareShutdown: async () => {
order.push('prepare:start');
await new Promise((resolve) => setTimeout(resolve, 200));
order.push('prepare:done');
},
onServerShutdown: () => {
order.push('shutdown');
},
};
await server.prepareShutdown({ drainMs: 0 });
// The listener's close callback starts the prepare hooks.
await new Promise((resolve) => setTimeout(resolve, 50));
await server.shutdown();
expect(order).toEqual(['prepare:start', 'prepare:done', 'shutdown']);
});
});
/**
* An extension may take over `services.jobs` (a fleet-wide scheduler). Core
* services register their jobs on it at start, so it must keep the built-in's
* first place in the layer.
*/
describe('PuterServer jobs replacement', () => {
it('lets an extension service registered as `jobs` replace the built-in and keep its first position', async () => {
const registered: string[] = [];
class ReplacementJobs extends JobScheduler {
override register(name: string, definition: JobDefinition) {
registered.push(name);
super.register(name, definition);
}
}
const started = vi.spyOn(JobScheduler.prototype, 'onServerStart');
extension.registerService('jobs', ReplacementJobs as never);
let server: PuterServer | undefined;
try {
server = await setupTestServer();
const { jobs } = server.services;
expect(jobs).toBeInstanceOf(ReplacementJobs);
expect(Object.keys(server.services)[0]).toBe('jobs');
expect(extension.import('service').jobs).toBeInstanceOf(
ReplacementJobs,
);
// Only the replacement gets lifecycle hooks.
expect(started.mock.contexts).toEqual([jobs]);
expect(registered).toContain('auth.sessions.purge');
} finally {
delete (extensionStore.services as Record<string, unknown>).jobs;
started.mockRestore();
await server?.shutdown();
}
});
});
/**
Loaded 100 of 559 files, more files were not shown because too many files have changed in this diff. Show more