mirror of
https://github.com/HeyPuter/puter.git
synced 2026-11-13 20:12:57 -10:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39dcb26f3f | ||
|
|
df9ce2c76f | ||
|
|
f61b3ea546 | ||
|
|
87b295f564 |
No files matched your search
@@ -47,13 +47,3 @@ jobs:
|
||||
PUTER_TEST_AI_MOONSHOT_API_KEY: ${{ secrets.PUTER_TEST_AI_MOONSHOT_API_KEY }}
|
||||
PUTER_TEST_AI_ZAI_API_KEY: ${{ secrets.PUTER_TEST_AI_ZAI_API_KEY }}
|
||||
PUTER_TEST_AI_ELEVENLABS_API_KEY: ${{ secrets.PUTER_TEST_AI_ELEVENLABS_API_KEY }}
|
||||
|
||||
# Realtime conversations through the SDK against live OpenAI. Builds
|
||||
# the SDK bundle these tests load itself, so it runs even when the
|
||||
# step above fails; the tests skip themselves without the key.
|
||||
- name: Run live realtime conversation tests
|
||||
if: ${{ !cancelled() }}
|
||||
run: npm run setupExtensions && (cd src/puter-js && npm run build) && npx vitest run --config src/puter-js/tests/api/vitest.config.ts src/puter-js/tests/api/realtime/convoLive.test.ts
|
||||
env:
|
||||
CI: 'true'
|
||||
PUTER_TEST_AI_OPENAI_API_KEY: ${{ secrets.PUTER_TEST_AI_OPENAI_API_KEY }}
|
||||
@@ -118,13 +118,9 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# The base branch lives in this repo; for a fork PR, the head
|
||||
# repo's copy of it can be arbitrarily stale.
|
||||
- ref: ${{ github.base_ref }}
|
||||
repository: ${{ github.repository }}
|
||||
artifact: base
|
||||
- ref: ${{ github.head_ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
artifact: pr
|
||||
|
||||
steps:
|
||||
@@ -133,7 +129,7 @@ jobs:
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ matrix.ref }}
|
||||
repository: ${{ matrix.repository }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
|
||||
- name: Set up Node.js
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
|
||||
@@ -201,6 +201,6 @@ Recipes are tested, copy-pasteable patterns for a task, one layer above the per-
|
||||
|
||||
- **Reuse existing UI primitives before writing new ones.** Windows and dialogs are `UIWindow*` functions in [src/gui/src/UI/](src/gui/src/UI/); generic pieces already exist (`UIAlert`, `UIPrompt`, `UIContextMenu`, `UINotification`, `UIPopover`, widgets in [UI/Components/](src/gui/src/UI/Components/)). A new window should read like its neighbors: an async function taking an options object, composing an HTML string, wiring behavior with jQuery, delegating to `UIWindow(...)`.
|
||||
- **Match the visual language.** Use existing CSS classes (`button`, `button-primary`, window chrome, form styles) and copy the layout patterns of neighboring windows; new styles go in [src/gui/src/css/](src/gui/src/css/) following existing conventions. Verify anything positional (menus, overlays, z-index) on both desktop and mobile viewports.
|
||||
- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js).
|
||||
- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js). Run `npm run check-translations` before opening the PR.
|
||||
- **Shared logic goes in helpers/services.** Reusable non-UI logic belongs in [src/gui/src/helpers/](src/gui/src/helpers/) or [src/gui/src/services/](src/gui/src/services/), not copy-pasted between windows.
|
||||
- **Tests.** Vitest is wired for the GUI (`src/**/*.test.js`; see [appOrder.test.js](src/gui/src/UI/Dashboard/appOrder.test.js) for the shape). Extract pure logic into functions and test those. Desktop behavior driven through puter.js (`puter.ui.*`) is covered by the Playwright harness in [src/puter-js/tests/e2e/](src/puter-js/tests/e2e/) — add a spec there when you change how the desktop renders SDK-driven UI.
|
||||
+1
-7
@@ -73,12 +73,6 @@ COPY --from=build --chown=node:node /opt/puter .
|
||||
RUN mkdir -p /etc/puter /var/puter && \
|
||||
chown -R node:node /etc/puter /var/puter
|
||||
|
||||
# Fix outdated HOME_URL in /etc/os-release (see issue #4329)
|
||||
# The base image may contain a reference to the old phoenix repository.
|
||||
RUN if [ -f /etc/os-release ]; then \
|
||||
sed -i 's|HOME_URL="https://github.com/HeyPuter/phoenix"|HOME_URL="https://github.com/HeyPuter/puter"|' /etc/os-release; \
|
||||
fi
|
||||
|
||||
# Self-hosters mount their override at this exact path. The v2 loader
|
||||
# deep-merges it over config.default.json (see backend/index.ts).
|
||||
ENV PUTER_CONFIG_PATH=/etc/puter/config.json
|
||||
@@ -93,4 +87,4 @@ USER node
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=30s --retries=3 \
|
||||
CMD wget --no-verbose --tries=1 --spider http://puter.localhost:4100/test || exit 1
|
||||
|
||||
CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
|
||||
CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
|
||||
@@ -350,9 +350,6 @@
|
||||
// not, a row past this is swept. Off by default; any value above 0 arms
|
||||
// the sweep.
|
||||
// "notificationRetentionDays": 14,
|
||||
//
|
||||
// Ms a live notification waits for open tabs to acknowledge it.
|
||||
// "notificationAckTimeoutMs": 5000,
|
||||
|
||||
// ── Alarms / alerting ───────────────────────────────────────────────
|
||||
// Where system alarms go. Severity is the routing decision — each
|
||||
|
||||
@@ -47,7 +47,7 @@ Both are supported ways to define an API. **Prefer a controller when you need fi
|
||||
|
||||
Provider sign-in requires the account's authenticator or a recovery code when 2FA is enabled, including signup callbacks that resolve an existing account. The five-minute challenge uses the same single-use `/login/otp` and `/login/recovery-code` verification as password sign-in. The GUI carries it in a URL fragment and removes it before initialization. A session and successful popup-return proof are issued only after verification; cancelling retains the popup's signed retry context without marking it authenticated.
|
||||
|
||||
Disabling 2FA through `/user-protected/disable-2fa` requires both password or OIDC revalidation and either the current six-digit authenticator `code` or an unused recovery code in `code`. Recovery codes are consumed atomically. Missing or incorrect codes leave the secret and remaining recovery codes intact. A team's requirement for 2FA still prevents disabling it.
|
||||
Disabling 2FA through `/user-protected/disable-2fa` requires both password or OIDC revalidation and the current six-digit authenticator `code`. Missing, malformed, or incorrect codes leave the secret and recovery codes intact. A team's requirement for 2FA still prevents disabling it.
|
||||
|
||||
Email changes through `/user-protected/change-email` require the password or OIDC revalidation cookie, plus a six-digit `code` from the authenticator when 2FA is enabled. Missing or malformed codes return `400 bad_request`; incorrect codes return `401 code_mismatch`. Both include `factor: 'otp'`. Confirmation links are bound to that verified request. A 2FA-enabled account must request a new link if its existing link was issued without second-factor verification.
|
||||
|
||||
|
||||
@@ -59,10 +59,6 @@ store:
|
||||
row — and the query seeks past it (`WHERE (col, id) > (?, ?) ORDER BY col,
|
||||
id`). Any SQL list gaining a cursor must have a deterministic `ORDER BY`
|
||||
ending in a unique tiebreaker (`id`).
|
||||
- The notification catch-up (`GET /events/fetch`) holds two keysets in one
|
||||
sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
|
||||
among rows up to that id. Each is its own indexed read, merged into one
|
||||
page, so a row replaced after a client read it comes back from that cursor.
|
||||
|
||||
Cursors that carry a sort also pin it: a request that passes a cursor plus a
|
||||
conflicting sort is rejected with 400.
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
# Browser push client
|
||||
|
||||
Puter's desktop shows pushes through `/sw.js`. Core registers it, and shows Settings → Notifications, only when the `pushEnabled` gui param is `true`; core never sets it. A backend extension that delivers pushes sets it with `services.homepage.setGuiParam('pushEnabled', true)` and can load its own code into the worker with `services.homepage.registerServiceWorkerScript(path)`.
|
||||
|
||||
Core subscribes the browser to a push service only when the server also publishes `pushWebPushKey` (see [Built-in Web Push client](#built-in-web-push-client)), and it has no `/push/*` routes of its own. The device, app, preference and subscription endpoints below come from that extension. On a deployment without one, such as most self-hosted servers, they answer core's plain 404 (`code: 'not_found'`), which `puter.notifications` reports as `push_unavailable` and the Notifications tab shows as unavailable. A device the extension can't find is `device_not_found`.
|
||||
|
||||
## Push payload
|
||||
|
||||
`sw.js` reads the push's JSON body. The message is its `puter` field, or `data.puter` for services that only carry string data, as an object or a JSON string:
|
||||
|
||||
```ts
|
||||
{ uid?: string; title?: string; body?: string; icon?: string; group?: string;
|
||||
target?: { app?: string; path?: string }; data?: unknown; channel?: string;
|
||||
type?: 'challenge'; challenge?: string }
|
||||
```
|
||||
|
||||
`uid` is the notification's uid, a later push with the same `group` replaces the one showing, and `target` and `data` reach the app when the user taps it.
|
||||
|
||||
When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so the page's push client registers the new subscription.
|
||||
|
||||
## Receipts
|
||||
|
||||
Before a notification is pushed, the server asks the open clients whether they received it live. It sends `notif.receipt` with `{ uid }` on the user's sockets for the app that sent it, or on the desktop's sockets for Puter's own notifications. The SDK remembers the uids its page received through `onNotification()`, `onPush()` or the desktop's feed, and answers once its page has that notification, or with `{ received: false }` when it never gets it. The server waits up to 5 seconds (`notificationAckTimeoutMs`).
|
||||
|
||||
An answer carries `visibilityState`, `focused` and `showsOwn`, the last being whether the page shows the notification itself. A website page shows it itself when it is hidden or unfocused, has an `onPush()` handler and notification permission, and no other tab of the same site is focused. Inside the desktop, the desktop shows it, and an app's `onPush()` result is never shown as a browser notification.
|
||||
|
||||
- An answer from a page that is visible and focused, or that shows the notification itself, stops the push.
|
||||
- Otherwise the notification is pushed. A website's pushes go only to the browsers that subscribed through its own worker, never to Puter's `/sw.js`. An app inside the desktop pushes through `/sw.js`, which shows the app's name and icon.
|
||||
- A hidden desktop tab shows no banner, so answers only from hidden desktop tabs don't stop a push of Puter's own notifications.
|
||||
- A client that can't answer, such as an older puter.js, holds the push back while it is connected.
|
||||
|
||||
## Registering
|
||||
|
||||
The registry is in [src/gui/src/helpers/pushClient.js](../src/gui/src/helpers/pushClient.js). A service script registers its client like this:
|
||||
|
||||
```js
|
||||
service_script(async (api) => {
|
||||
api.use()['push.registerPushClient']?.(client);
|
||||
});
|
||||
```
|
||||
|
||||
`use(id)` throws when nothing is registered under `id`, so the form above also runs on a Puter version without the registry and does nothing there. Any script that runs after the GUI bundle can call `use()['push.registerPushClient']?.(client)` the same way. Registering again replaces the client. Registering dispatches `push-client-registered` on `window`, and an open Notifications tab redraws.
|
||||
|
||||
With no client registered, the tab says push notifications aren't available in this browser.
|
||||
|
||||
### Built-in Web Push client
|
||||
|
||||
When the server also sets the `pushWebPushKey` gui param (a VAPID public key), core registers its own client ([webPushClient.js](../src/gui/src/helpers/webPushClient.js)) unless an extension registered one first. It uses `puter.notifications.enable()` / `disable()`, the same standard Web Push flow apps use on their own origins:
|
||||
|
||||
1. `GET /push/web/key` for the key, then `pushManager.subscribe()` on the `/sw.js` registration.
|
||||
2. `POST /push/web/subscriptions { subscription }`. A subscription the server already has answers `{ status: 'active' }`. A new one answers `{ status: 'pending' }`, and the server sends it a push with `{ type: 'challenge', challenge }`.
|
||||
3. The worker shows that push and posts `{ type: 'puter-push-challenge', challenge }` to the origin's windows; the page sends it to `POST /push/web/subscriptions/verify`. The device is registered once the page hands the confirmation back.
|
||||
4. `DELETE /push/web/subscriptions { endpoint }` removes it.
|
||||
|
||||
Apps on their own origin do the same through `push-sw.js`, which they import into their own worker. A subscription made with an app token is the app's: its pushes are that app's notifications only.
|
||||
|
||||
## Interface
|
||||
|
||||
```ts
|
||||
interface PushClient {
|
||||
enable(): Promise<{ deviceUid: string }>;
|
||||
disable(): Promise<void>;
|
||||
status(): Promise<{
|
||||
permission: 'default' | 'granted' | 'denied';
|
||||
registered: boolean;
|
||||
deviceUid?: string;
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. The built-in client runs `puter.notifications.enable()`, which subscribes with the `/sw.js` registration and registers the browser through the Web Push flow above (`platform: 'webpush'`). It then resolves the `uid` of this browser's `webpush` device from `listDevices()`. It rejects when permission is refused or the browser wasn't registered.
|
||||
- `disable()` runs `puter.notifications.disable()`, which sends `DELETE /push/web/subscriptions { endpoint }` and ends the browser's subscription. The tab calls it for Turn off, and for Remove on the device marked as this browser. Remove on any other device is `DELETE /push/devices/:uid`.
|
||||
- `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user, and `deviceUid` is that device's `uid`.
|
||||
|
||||
After `enable()` resolves, the tab calls `POST /push/devices/:uid/test` and asks the user whether a notification showed up.
|
||||
|
||||
The tab also reads `GET /push/apps` and `GET`/`POST /push/preferences`.
|
||||
|
||||
Core keeps its own record of a browser the user turned off, `puter.push.browser-off:<user uuid>` in `localStorage`, written by Turn off and by Remove on this browser's device, and cleared when `enable()` resolves. The silent registration above skips a browser with that record, so a device removed here stays removed while one removed from another device, or a new sign-in in a browser that already allows notifications, registers again.
|
||||
+1
-19
@@ -12,17 +12,7 @@ Generates secrets, writes `.env` + `puter/config/config.json`, downloads `docker
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Docker** with the `compose` plugin. This is the one thing you have to install
|
||||
yourself — the installer checks for it and stops with an explanation rather
|
||||
than installing it for you.
|
||||
- **macOS / Windows:** [Docker Desktop](https://docs.docker.com/get-docker/).
|
||||
Both ship the `compose` plugin.
|
||||
- **Linux:** [Docker Engine](https://docs.docker.com/engine/install/) plus the
|
||||
[`docker-compose-plugin`](https://docs.docker.com/compose/install/) package.
|
||||
- Confirm before running anything else: `docker compose version` should print a
|
||||
version, not `command not found`.
|
||||
- If Docker is installed but the shell can't see it, make sure your user is in
|
||||
the `docker` group (Linux) or that Docker Desktop is actually running.
|
||||
- **Docker** with the `compose` plugin.
|
||||
- A **domain** with DNS access — you need a wildcard record (`*.your-domain.com` → server IP). Puter routes by subdomain (`api.<domain>`, `site.<domain>`, `app.<domain>`).
|
||||
- Optional: **TLS certs** (or `certbot` to grab them — see Step 3).
|
||||
|
||||
@@ -667,14 +657,6 @@ Migrations re-apply idempotently across pulls. Volumes are preserved.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**`missing required command: docker` — the installer stops immediately.**
|
||||
Docker isn't on your `PATH` yet. Puter self-hosting is a Docker Compose stack, so
|
||||
nothing can run without it. Install [Docker Desktop](https://docs.docker.com/get-docker/)
|
||||
on macOS/Windows or [Docker Engine](https://docs.docker.com/engine/install/) plus
|
||||
`docker-compose-plugin` on Linux, then confirm `docker compose version` works in
|
||||
the same shell and re-run the installer. On Linux, a fresh Docker install usually
|
||||
needs a new login session before your user picks up the `docker` group.
|
||||
|
||||
**Site loads but I get a 502 / "Bad Gateway" from Caddy.**
|
||||
The puter container failed to come up. `docker compose logs puter` will tell you which dependency rejected it (most often DB password mismatch between `.env` and `config.json`).
|
||||
|
||||
|
||||
@@ -314,41 +314,6 @@ describe('thumbnails extension — handleThumbnailRead', () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
it('keeps read URLs stable within an hour and rotates them at the next hour', async () => {
|
||||
const key = mintedKey();
|
||||
const signedAt = Date.UTC(2026, 9, 26, 10);
|
||||
const read = async () => {
|
||||
const entry: Record<string, unknown> = {
|
||||
thumbnail: `s3://${BUCKET}/${key}`,
|
||||
};
|
||||
await handleThumbnailRead(entry, {
|
||||
s3,
|
||||
s3Presign,
|
||||
bucketName: BUCKET,
|
||||
bucketEndpoint: BUCKET_ENDPOINT,
|
||||
fsEntry: server.stores.fsEntry,
|
||||
});
|
||||
return entry.thumbnail as string;
|
||||
};
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
try {
|
||||
vi.setSystemTime(signedAt + 1000);
|
||||
const first = await read();
|
||||
vi.setSystemTime(signedAt + 3599_000);
|
||||
expect(await read()).toBe(first);
|
||||
expect(new URL(first).searchParams.get('X-Amz-Date')).toBe(
|
||||
'20261026T100000Z',
|
||||
);
|
||||
expect(new URL(first).searchParams.get('X-Amz-Expires')).toBe(
|
||||
'604800',
|
||||
);
|
||||
vi.setSystemTime(signedAt + 3600_000);
|
||||
expect(await read()).not.toBe(first);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
// Thumbnails stored before keys were bound to entries keep displaying.
|
||||
it.each([
|
||||
['an entry-bound key', () => mintedKey()],
|
||||
|
||||
@@ -307,9 +307,6 @@ export const handleThumbnailRead = async (
|
||||
const thumb = entry.thumbnail;
|
||||
if (typeof thumb !== 'string' || !thumb) return;
|
||||
const presignClient = deps.s3Presign;
|
||||
const signingDate = new Date(
|
||||
Math.floor(Date.now() / 3_600_000) * 3_600_000,
|
||||
);
|
||||
|
||||
if (isStoragePointer(thumb, deps.bucketEndpoint)) {
|
||||
const key = resolveThumbnailKey(thumb);
|
||||
@@ -322,7 +319,7 @@ export const handleThumbnailRead = async (
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
{ expiresIn: 604800, signingDate },
|
||||
{ expiresIn: 604800 },
|
||||
);
|
||||
} else if (thumb.startsWith('data')) {
|
||||
// Move an inline thumbnail into storage. The row only takes the new
|
||||
@@ -376,7 +373,7 @@ export const handleThumbnailRead = async (
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
{ expiresIn: 604800, signingDate },
|
||||
{ expiresIn: 604800 },
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
+1
-21
@@ -53,29 +53,9 @@ need() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
|
||||
}
|
||||
|
||||
check_docker() {
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
echo >&2 ""
|
||||
echo >&2 "[puter-install] missing required command: docker"
|
||||
echo >&2 ""
|
||||
echo >&2 "Docker is required to run Puter self-hosted."
|
||||
echo >&2 ""
|
||||
echo >&2 "Installation instructions:"
|
||||
echo >&2 " - macOS (Apple Silicon & Intel): https://docs.docker.com/desktop/setup/install/mac-install/"
|
||||
echo >&2 " (Docker Desktop is recommended. Alternatives: OrbStack, Colima)"
|
||||
echo >&2 " - Linux: https://docs.docker.com/engine/install/"
|
||||
echo >&2 " - Windows: https://docs.docker.com/desktop/setup/install/windows-install/"
|
||||
echo >&2 ""
|
||||
echo >&2 "After installing Docker, make sure it's running and try again."
|
||||
echo >&2 ""
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Step 1: dependency check ────────────────────────────────────────
|
||||
log "checking dependencies"
|
||||
check_docker
|
||||
need docker
|
||||
need curl
|
||||
need openssl
|
||||
docker compose version >/dev/null 2>&1 \
|
||||
|
||||
Generated
+768
-786
File diff suppressed because it is too large.
Load diff
+2
-2
@@ -54,6 +54,7 @@
|
||||
"dev": "npm start",
|
||||
"build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
|
||||
"build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build",
|
||||
"check-translations": "node tools/check-translations.js",
|
||||
"prepare": "husky",
|
||||
"build:ts": "tsc -p tsconfig.build.json && node ./tools/write-dist-package-json.mjs",
|
||||
"check:puterjs:types": "node tools/checkPuterjsTypes.mjs",
|
||||
@@ -78,8 +79,7 @@
|
||||
"javascript-time-ago": "^2.5.11",
|
||||
"libphonenumber-js": "1.13.15",
|
||||
"miniflare": "^4.20260617.1",
|
||||
"open": "^10.1.0",
|
||||
"sharp": "^0.35.5"
|
||||
"open": "^10.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=24.0.0"
|
||||
|
||||
@@ -108,25 +108,6 @@ describe('AbstractDatabaseClient — unimplemented surface', () => {
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — replica-aware reads', () => {
|
||||
it('avoids a primary read when the replica has rows in sequential mode', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
expect(
|
||||
await client.tryHardRead('SELECT 1', [], { race: false }),
|
||||
).toEqual([{ id: 1 }]);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([[], new Error('replica unavailable')])(
|
||||
'falls back once in sequential mode',
|
||||
async (replica) => {
|
||||
const client = new FixtureDatabaseClient(replica, [{ id: 2 }]);
|
||||
expect(
|
||||
await client.tryHardRead('SELECT 1', [], { race: false }),
|
||||
).toEqual([{ id: 2 }]);
|
||||
expect(client.preads).toEqual(['SELECT 1']);
|
||||
},
|
||||
);
|
||||
|
||||
it('returns replica rows without waiting on the primary', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
@@ -275,28 +256,6 @@ describe('AbstractDatabaseClient — SQL generation', () => {
|
||||
expect(postgres().timestampFromUnixSql()).toBe('to_timestamp(?)');
|
||||
});
|
||||
|
||||
it('reads a timestamp column back as unix seconds in each engine', () => {
|
||||
expect(sqlite().unixSecondsSql('`t`')).toBe(
|
||||
"CAST(strftime('%s', `t`) AS INTEGER)",
|
||||
);
|
||||
expect(mysql().unixSecondsSql('`t`')).toBe('UNIX_TIMESTAMP(`t`)');
|
||||
expect(postgres().unixSecondsSql('`t`')).toBe(
|
||||
'EXTRACT(EPOCH FROM `t`)::bigint',
|
||||
);
|
||||
});
|
||||
|
||||
it('reads the database clock as epoch milliseconds in each dialect', () => {
|
||||
expect(sqlite().nowMsExpr()).toBe(
|
||||
"CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
|
||||
);
|
||||
expect(mysql().nowMsExpr()).toBe(
|
||||
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
|
||||
);
|
||||
expect(postgres().nowMsExpr()).toBe(
|
||||
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
|
||||
);
|
||||
});
|
||||
|
||||
it('describes a stored timestamp the way each driver reads it back', () => {
|
||||
const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
|
||||
expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');
|
||||
|
||||
@@ -138,27 +138,25 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
|
||||
/**
|
||||
* Like `read()`, but an empty or failed replica read falls back to the
|
||||
* primary, so replication lag can't hide a row that was just written. By
|
||||
* default the primary read starts alongside the replica one; `race: false`
|
||||
* reads the primary only after the replica returns empty or fails. Without
|
||||
* a replica this is a single `read()`.
|
||||
* primary, so replication lag can't hide a row that was just written. The
|
||||
* primary read starts alongside the replica one rather than after it.
|
||||
* Without a replica this is a single `read()`.
|
||||
*/
|
||||
@Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
|
||||
async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
{ race = true }: { race?: boolean } = {},
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (!this.hasReadReplica()) return this.read(query, params);
|
||||
|
||||
const primary = race ? settle(this.pread(query, params)) : null;
|
||||
const primary = settle(this.pread(query, params));
|
||||
try {
|
||||
const rows = await this.replicaRead(query, params);
|
||||
if (rows.length > 0) return rows;
|
||||
} catch {
|
||||
// replica failed — fall through to primary
|
||||
}
|
||||
const result = await (primary ?? settle(this.pread(query, params)));
|
||||
const result = await primary;
|
||||
if ('error' in result) throw result.error;
|
||||
return result.value;
|
||||
}
|
||||
@@ -388,34 +386,6 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A timestamp column as unix seconds, computed by the database: the mysql
|
||||
* driver reads a stored UTC timestamp as local time, so parsing it in JS
|
||||
* can be off by the server's UTC offset.
|
||||
*/
|
||||
unixSecondsSql(column: string): string {
|
||||
return this.case({
|
||||
postgres: `EXTRACT(EPOCH FROM ${column})::bigint`,
|
||||
mysql: `UNIX_TIMESTAMP(${column})`,
|
||||
otherwise: `CAST(strftime('%s', ${column}) AS INTEGER)`,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The database's own clock as epoch milliseconds, so nodes whose clocks
|
||||
* disagree still agree on it. The mysql form ignores the session time zone,
|
||||
* so a DST change can't shift it.
|
||||
*/
|
||||
nowMsExpr(): string {
|
||||
return this.case({
|
||||
sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
|
||||
postgres:
|
||||
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
|
||||
otherwise:
|
||||
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* What a read of a timestamp column holding `seconds` returns: sqlite gives
|
||||
* back the UTC text it stored, mysql2 and pg a `Date` read in node's time
|
||||
|
||||
@@ -217,8 +217,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
return this.configuration === Configuration.REPLICA;
|
||||
}
|
||||
|
||||
// No failover: `tryHardRead` falls back to the primary on an empty result
|
||||
// or error (raced alongside by default, after the replica with `race: false`).
|
||||
// No failover: `tryHardRead` already has a primary read in flight.
|
||||
protected override async replicaRead(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
|
||||
@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
|
||||
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
|
||||
|
||||
/** Highest schema version the migration table can reach. */
|
||||
const CURRENT_SCHEMA_VERSION = 96;
|
||||
const CURRENT_SCHEMA_VERSION = 92;
|
||||
|
||||
/**
|
||||
* These suites migrate real files on disk. Idle they finish in well under a
|
||||
@@ -673,38 +673,6 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
|
||||
});
|
||||
});
|
||||
|
||||
it('adds kv_share_handles.notify from 0099, off for every existing share', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-notify-'));
|
||||
const path = join(dir, 'puter.sqlite');
|
||||
try {
|
||||
const before = new SqliteDatabaseClient(
|
||||
sqliteConfig({ inMemory: false, path, targetVersion: 94 }),
|
||||
);
|
||||
await before.onServerStart();
|
||||
await before.write(
|
||||
'INSERT INTO `kv_share_handles` (`handle`, `owner_user_id`, ' +
|
||||
'`grantee_user_id`, `app_uid`, `key_prefix`, `permission`, ' +
|
||||
'`created_at`) VALUES (?, 1, 1, ?, ?, ?, 0)',
|
||||
['kvh-before-notify', 'os-global', 'a:', 'kv-share:x'],
|
||||
);
|
||||
before.onServerShutdown();
|
||||
|
||||
const after = new SqliteDatabaseClient(
|
||||
sqliteConfig({ inMemory: false, path }),
|
||||
);
|
||||
await after.onServerStart();
|
||||
await expect(
|
||||
after.read(
|
||||
'SELECT `notify` FROM `kv_share_handles` WHERE `handle` = ?',
|
||||
['kvh-before-notify'],
|
||||
),
|
||||
).resolves.toEqual([{ notify: 0 }]);
|
||||
after.onServerShutdown();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('indexes notification.created_at for the retention sweep', async () => {
|
||||
const rows = await client.read('PRAGMA index_list(`notification`)');
|
||||
expect(rows.map((r) => String(r.name))).toContain(
|
||||
@@ -712,29 +680,6 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
|
||||
);
|
||||
});
|
||||
|
||||
it('indexes notification.replaced_at per mailbox slice for catch-up', async () => {
|
||||
const columns = await client.read(
|
||||
'PRAGMA index_info(`idx_notification_replaced`)',
|
||||
);
|
||||
expect(columns.map((c) => String(c.name))).toEqual([
|
||||
'user_id',
|
||||
'audience',
|
||||
'app_uid',
|
||||
'replaced_at',
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps notification.send_key unique', async () => {
|
||||
const rows = await client.read('PRAGMA index_list(`notification`)');
|
||||
expect(
|
||||
rows.find((r) => r.name === 'idx_notification_send_key'),
|
||||
).toMatchObject({ unique: 1 });
|
||||
const columns = await client.read(
|
||||
'PRAGMA index_info(`idx_notification_send_key`)',
|
||||
);
|
||||
expect(columns.map((c) => String(c.name))).toEqual(['send_key']);
|
||||
});
|
||||
|
||||
it('leaves an already-migrated database untouched on a second boot', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
|
||||
const path = join(dir, 'nested', 'puter.sqlite');
|
||||
|
||||
@@ -131,10 +131,6 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
|
||||
[89, ['0094_deleted-apps.sql']],
|
||||
[90, ['0095_kv-share-handle-levels.sql']],
|
||||
[91, ['0096_kv-share-handle-quota-indexes.sql']],
|
||||
[92, ['0097_user-to-user-holder-permission-index.sql']],
|
||||
[93, ['0098_notification-groups.sql']],
|
||||
[94, ['0099_kv-share-handle-notify.sql']],
|
||||
[95, ['0100_notification-send-key.sql']],
|
||||
];
|
||||
|
||||
export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Lets a holder's grant check read only the permissions it names. Guarded, so
|
||||
-- a replay does nothing.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_u2u_holder_permission_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_u2u_holder_permission_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user_to_user_permissions'
|
||||
AND INDEX_NAME = 'idx_user_to_user_permissions_holder_permission'
|
||||
) THEN
|
||||
ALTER TABLE `user_to_user_permissions` ADD INDEX `idx_user_to_user_permissions_holder_permission` (`holder_user_id`, `permission`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_u2u_holder_permission_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_u2u_holder_permission_index;
|
||||
@@ -1,53 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- `notification`.`group_key` and `replaced_at`. See
|
||||
-- sqlite/0098_notification-groups.sql for the rationale. The key is a hex
|
||||
-- digest, so ascii and compared byte for byte. No per-file applied-state
|
||||
-- tracking, so every step is guarded and a replay does nothing.
|
||||
|
||||
CALL _puter_add_col('notification', 'group_key', '`group_key` char(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
|
||||
CALL _puter_add_col('notification', 'replaced_at', '`replaced_at` bigint DEFAULT NULL');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_group_indexes;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_notification_group_indexes()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND INDEX_NAME = 'idx_notification_group'
|
||||
) THEN
|
||||
ALTER TABLE `notification` ADD INDEX `idx_notification_group`
|
||||
(`user_id`, `app_uid`, `group_key`);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND INDEX_NAME = 'idx_notification_replaced'
|
||||
) THEN
|
||||
ALTER TABLE `notification` ADD INDEX `idx_notification_replaced`
|
||||
(`user_id`, `audience`, `app_uid`, `replaced_at`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_notification_group_indexes();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_group_indexes;
|
||||
@@ -1,23 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Notifications through a share handle. See
|
||||
-- sqlite/0099_kv-share-handle-notify.sql for the column. No per-file
|
||||
-- applied-state tracking, so it goes through _puter_add_col and a replay does
|
||||
-- nothing.
|
||||
|
||||
CALL _puter_add_col('kv_share_handles', 'notify', '`notify` tinyint(1) NOT NULL DEFAULT 0');
|
||||
@@ -1,43 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- `notification`.`send_key`. See sqlite/0100_notification-send-key.sql for the
|
||||
-- rationale. The key is a uuid, so ascii and compared byte for byte. No
|
||||
-- per-file applied-state tracking, so every step is guarded and a replay does
|
||||
-- nothing.
|
||||
|
||||
CALL _puter_add_col('notification', 'send_key', '`send_key` char(36) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_notification_send_key_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND INDEX_NAME = 'idx_notification_send_key'
|
||||
) THEN
|
||||
ALTER TABLE `notification` ADD UNIQUE KEY `idx_notification_send_key`
|
||||
(`send_key`), ALGORITHM=INPLACE, LOCK=NONE;
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_notification_send_key_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_send_key_index;
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Lets a holder's grant check read only the permissions it names.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_to_user_permissions_holder_permission
|
||||
ON user_to_user_permissions (holder_user_id, permission);
|
||||
@@ -1,29 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- notification.group_key and replaced_at. See
|
||||
-- sqlite/0098_notification-groups.sql for the rationale. Idempotent via IF NOT
|
||||
-- EXISTS.
|
||||
|
||||
ALTER TABLE notification ADD COLUMN IF NOT EXISTS group_key varchar(64) DEFAULT NULL;
|
||||
ALTER TABLE notification ADD COLUMN IF NOT EXISTS replaced_at bigint DEFAULT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_group
|
||||
ON notification (user_id, app_uid, group_key);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_replaced
|
||||
ON notification (user_id, audience, app_uid, replaced_at);
|
||||
@@ -1,22 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Notifications through a share handle. See
|
||||
-- sqlite/0099_kv-share-handle-notify.sql for the column. Idempotent via
|
||||
-- IF NOT EXISTS; there is no per-file applied-state tracking.
|
||||
|
||||
ALTER TABLE kv_share_handles ADD COLUMN IF NOT EXISTS notify smallint NOT NULL DEFAULT 0;
|
||||
@@ -1,24 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- notification.send_key. See sqlite/0100_notification-send-key.sql for the
|
||||
-- rationale. Idempotent via IF NOT EXISTS.
|
||||
|
||||
ALTER TABLE notification ADD COLUMN IF NOT EXISTS send_key char(36) DEFAULT NULL;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_send_key
|
||||
ON notification (send_key);
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Lets a holder's grant check read only the permissions it names.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_user_to_user_permissions_holder_permission`
|
||||
ON `user_to_user_permissions` (`holder_user_id`, `permission`);
|
||||
@@ -1,38 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- What app message groups need from `notification`:
|
||||
--
|
||||
-- - `group_key` : a sha256 hex digest of an app message's `group`, so the
|
||||
-- notification a newer one replaces is found by index.
|
||||
-- Null on every other row, and on rows written before
|
||||
-- this, which therefore never match.
|
||||
-- - `replaced_at` : epoch milliseconds of the last replacement, null for a
|
||||
-- row never replaced. Live and fetched copies carry it as
|
||||
-- their version, so a client can tell a replacement from a
|
||||
-- copy it already has. Indexed per mailbox slice, so a
|
||||
-- catch-up reads the rows replaced since its cursor
|
||||
-- without scanning the ones that never were.
|
||||
|
||||
ALTER TABLE `notification` ADD COLUMN `group_key` TEXT DEFAULT NULL;
|
||||
ALTER TABLE `notification` ADD COLUMN `replaced_at` INTEGER DEFAULT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_notification_group`
|
||||
ON `notification` (`user_id`, `app_uid`, `group_key`);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_notification_replaced`
|
||||
ON `notification` (`user_id`, `audience`, `app_uid`, `replaced_at`);
|
||||
@@ -1,23 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Whether the two parties of a share may send each other notifications through
|
||||
-- its handle. Off by default, so every existing share reads as not allowing
|
||||
-- them.
|
||||
|
||||
ALTER TABLE `kv_share_handles`
|
||||
ADD COLUMN `notify` INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -1,27 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- `notification`.`send_key`: the key of the delayed send (a `deferred`
|
||||
-- schedule) that wrote this row or last replaced it, derived from the
|
||||
-- schedule and its slot. Unique, so one slot can never write two rows, and a
|
||||
-- send that crashed after writing finds its row again by it. Null on every row
|
||||
-- a delayed send never touched.
|
||||
|
||||
ALTER TABLE `notification` ADD COLUMN `send_key` TEXT DEFAULT NULL;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS `idx_notification_send_key`
|
||||
ON `notification` (`send_key`);
|
||||
@@ -75,15 +75,6 @@ export type EventMap = {
|
||||
serverStart: Record<string, never>;
|
||||
serverPrepareShutdown: Record<string, never>;
|
||||
serverShutdown: Record<string, never>;
|
||||
/**
|
||||
* Emitted by a host extension that knows this server will restart soon.
|
||||
* Long-lived connections ask their clients to reconnect; nothing is refused
|
||||
* or closed, so a false alarm is harmless.
|
||||
*/
|
||||
serverRestartAnnounced: {
|
||||
/** How long until the server stops, when known. */
|
||||
closeInMs?: number;
|
||||
};
|
||||
|
||||
// ---- AI ----
|
||||
'ai.prompt.validate': {
|
||||
@@ -217,7 +208,7 @@ export type EventMap = {
|
||||
*/
|
||||
is_temp?: boolean;
|
||||
ip?: string | null;
|
||||
source?: 'oidc' | 'team_provision';
|
||||
source?: 'oidc';
|
||||
req?: unknown;
|
||||
data?: unknown;
|
||||
abuse?: unknown;
|
||||
@@ -277,7 +268,7 @@ export type EventMap = {
|
||||
ip?: string | null;
|
||||
fingerprint?: string | null;
|
||||
reputation?: number | null;
|
||||
source?: 'oidc' | 'team_provision';
|
||||
source?: 'oidc';
|
||||
requires_phone_verification: boolean;
|
||||
requires_card_verification: boolean;
|
||||
accepted: boolean;
|
||||
@@ -323,8 +314,6 @@ export type EventMap = {
|
||||
// the abuse extension cap sends per device (across accounts) so a
|
||||
// device farm can't dodge the per-account cap with fresh signups.
|
||||
device_fingerprint: string | null;
|
||||
// Browser-signals dispatch id the client sent, or null.
|
||||
dispatch_id: string | null;
|
||||
allowed: boolean;
|
||||
reason: string | null;
|
||||
// With `allowed` false, offer the card fallback in place of SMS. Only
|
||||
@@ -560,30 +549,9 @@ export type EventMap = {
|
||||
appUid: string | null;
|
||||
value: Record<string, unknown>;
|
||||
createdAt: number;
|
||||
/** An earlier notification replaced in place; `createdAt` is when. */
|
||||
replaced?: true;
|
||||
/** Events handler runs behind whatever created it. */
|
||||
handlerDepth?: number;
|
||||
};
|
||||
/**
|
||||
* Every non-silent write or update, only on the node that wrote it. The
|
||||
* value is the listener's own copy of what the socket carries.
|
||||
*/
|
||||
'notif.written': {
|
||||
userId: number;
|
||||
uid: string;
|
||||
type: string;
|
||||
audience: string;
|
||||
/** The row's app; null when it names none or an update could not tell. */
|
||||
appUid: string | null;
|
||||
value: Record<string, unknown>;
|
||||
/** An earlier notification rewritten in place. */
|
||||
updated?: true;
|
||||
/** Names this one send, for a receipt asked about it. */
|
||||
sendId: string;
|
||||
/** The same `sendId` may be emitted again, as a delayed send is. */
|
||||
replayable?: true;
|
||||
};
|
||||
/**
|
||||
* One notification, addressed at whatever sockets each region holds for the
|
||||
* recipient. `outer.*` reaches peer regions and never sibling nodes, so
|
||||
@@ -592,31 +560,9 @@ export type EventMap = {
|
||||
*/
|
||||
'outer.notif.delivery': {
|
||||
userId: number;
|
||||
wire: 'notif.message' | 'notif.unreads' | 'notif.ack' | 'notif.removed';
|
||||
wire: 'notif.message' | 'notif.unreads' | 'notif.ack';
|
||||
response: unknown;
|
||||
};
|
||||
/**
|
||||
* How one live notification went, on the node that asked for its receipt:
|
||||
* from the desktop's sockets, or with `appUid` the app's own.
|
||||
*/
|
||||
'notif.delivery.settled': {
|
||||
sendId?: string;
|
||||
uid: string;
|
||||
userId: number;
|
||||
type: string;
|
||||
/** The app whose own sockets were asked; absent for the desktop's. */
|
||||
appUid?: string;
|
||||
outcome: NotifDeliveryOutcome;
|
||||
/** A socket in this region acknowledges the receipt. */
|
||||
ackCapable: boolean;
|
||||
/** No socket here is an older client that acknowledges nothing. */
|
||||
everySocketAcks: boolean;
|
||||
/** Every node here answered the ask, not just this one. */
|
||||
complete: boolean;
|
||||
/** A peer region holds a socket for the user. */
|
||||
peerRegions: boolean;
|
||||
responders: NotifDeliveryResponder[];
|
||||
};
|
||||
|
||||
// ---- Metering ----
|
||||
// Recurring charges are pure mechanism here: the metering service knows
|
||||
@@ -758,7 +704,6 @@ export type EventMap = {
|
||||
'outer.gui.item.renamed': GuiEvent;
|
||||
'outer.gui.notif.ack': GuiEvent<{ uid: string }>;
|
||||
'outer.gui.notif.message': GuiEvent<{ uid: string; notification: unknown }>;
|
||||
'outer.gui.notif.removed': GuiEvent<{ uid: string }>;
|
||||
'outer.gui.notif.unreads': GuiEvent<{
|
||||
unreads: { uid: string; notification: unknown }[];
|
||||
}>;
|
||||
@@ -766,7 +711,7 @@ export type EventMap = {
|
||||
// ---- Subdomains ----
|
||||
// `uid` is the row uuid — for `delete` it is the only surviving handle a
|
||||
// listener can use to find state keyed to the row after it is gone.
|
||||
'subdomain.delete': { subdomain: string; uid?: string; userId: number };
|
||||
'subdomain.delete': { subdomain: string; uid?: string };
|
||||
'subdomain.update': { subdomain: string; uid?: string };
|
||||
'site.htmlServed': {
|
||||
subdomain: string;
|
||||
@@ -954,9 +899,6 @@ export type AiCostFactorEvent = {
|
||||
*/
|
||||
export type LifecyclePhase = 'before' | 'after' | 'error' | 'reject';
|
||||
|
||||
/** Status a vetoing `before` listener may answer with. */
|
||||
export type LifecycleRejectStatus = 403 | 429;
|
||||
|
||||
/**
|
||||
* Payload for `driver.<iface>.<method>.<phase>` events.
|
||||
*
|
||||
@@ -986,8 +928,6 @@ export type DriverMethodLifecycleEvent = {
|
||||
allow?: boolean;
|
||||
/** Optional human-readable reason surfaced to the caller when vetoed. */
|
||||
rejectReason?: string;
|
||||
/** Status for a veto: 403 (default) or 429 for a throttle. */
|
||||
rejectStatus?: LifecycleRejectStatus;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1022,8 +962,6 @@ export type RouteLifecycleEvent = {
|
||||
allow?: boolean;
|
||||
/** Optional human-readable reason surfaced to the caller when vetoed. */
|
||||
rejectReason?: string;
|
||||
/** Status for a veto: 403 (default) or 429 for a throttle. */
|
||||
rejectStatus?: LifecycleRejectStatus;
|
||||
};
|
||||
|
||||
export type EventKey = keyof EventMap & string;
|
||||
@@ -1032,11 +970,10 @@ export type EventKey = keyof EventMap & string;
|
||||
// Generates a wildcard for every non-final dot-separated prefix of K.
|
||||
export type WildcardPrefixes<K extends string> =
|
||||
K extends `${infer Head}.${infer Tail}`
|
||||
?
|
||||
| `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
? | `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
: never;
|
||||
|
||||
export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
|
||||
@@ -1045,25 +982,6 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
|
||||
? Extract<EventKey, `${Prefix}.${string}`>
|
||||
: P & EventKey;
|
||||
|
||||
/** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
|
||||
export type NotifDeliveryOutcome =
|
||||
| 'received'
|
||||
| 'unreceived'
|
||||
| 'no-socket'
|
||||
| 'remote-only';
|
||||
|
||||
/** One socket's receipt for a live notification. */
|
||||
export interface NotifDeliveryResponder {
|
||||
visibilityState: 'visible' | 'hidden' | 'unknown';
|
||||
/** The page's window has focus. Absent from older clients. */
|
||||
focused?: boolean;
|
||||
/**
|
||||
* The page shows its own browser notification while hidden or unfocused.
|
||||
* Absent from older clients.
|
||||
*/
|
||||
showsOwn?: boolean;
|
||||
}
|
||||
|
||||
export type EventMetadata = {
|
||||
from_outside?: boolean;
|
||||
/** Set by `emitAndWait` when a listener threw, so a verdict hook can refuse. */
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -43,20 +43,6 @@ import { PuterClient } from '../types.js';
|
||||
*
|
||||
* On a miss, a handler set here may deploy the script itself and retry once
|
||||
* with `x-puter-events-deployed: 1`, skipping the dispatcher's own callback.
|
||||
*
|
||||
* Calls that come due within a few milliseconds of each other go out together:
|
||||
*
|
||||
* POST <events dispatcher>/invoke-batch
|
||||
* x-puter-internal-auth: <internal secret>
|
||||
* { calls: [{ script, appUid, key, deployed?, body, waitMs }] }
|
||||
*
|
||||
* The dispatcher runs a few at a time, in order, and streams NDJSON: `{ i,
|
||||
* started: true }` when a call goes to its worker, then `{ i, status, handled?,
|
||||
* error?, retryAfter?, dispatch? }` when it finishes — the same facts the
|
||||
* single route carries in its status and headers. A call's timeout runs from
|
||||
* its `started` line, so time spent queued behind other calls is never the
|
||||
* handler's. A call still queued `waitMs` after the batch arrived is never
|
||||
* started and answers `dispatch: "queue-timeout"`.
|
||||
*/
|
||||
|
||||
/** The one route an events worker answers. */
|
||||
@@ -65,27 +51,6 @@ export const EVENTS_INVOKE_PATH = '/__events/invoke';
|
||||
/** The route the events dispatcher takes an invocation on. */
|
||||
export const EVENTS_DISPATCH_PATH = '/invoke';
|
||||
|
||||
/** The route it takes several on, answering each as it finishes. */
|
||||
export const EVENTS_DISPATCH_BATCH_PATH = '/invoke-batch';
|
||||
|
||||
/** Calls one batch carries at most. */
|
||||
export const INVOKE_BATCH_SIZE = 60;
|
||||
|
||||
/** How long the first call of a batch waits for others to join it. */
|
||||
export const INVOKE_BATCH_WINDOW_MS = 5;
|
||||
|
||||
/**
|
||||
* How long a dispatcher without the batch route is sent single calls before the
|
||||
* batch route is tried again.
|
||||
*/
|
||||
const BATCH_UNSUPPORTED_RETRY_MS = 5 * 60_000;
|
||||
|
||||
/**
|
||||
* How long past its wait a batched call's `started` line may take to arrive
|
||||
* before the call is given up as never started.
|
||||
*/
|
||||
const BATCH_START_GRACE_MS = 2_000;
|
||||
|
||||
/**
|
||||
* Header the dispatcher marks its own failures with. Anything carrying it never
|
||||
* reached a handler, so its status says nothing about the delivery.
|
||||
@@ -103,23 +68,6 @@ export const EVENTS_HANDLED_HEADER = 'x-puter-events-handled';
|
||||
/** Machine-readable reason on the runtime's own failure answers. */
|
||||
export const EVENTS_ERROR_HEADER = 'x-puter-events-error';
|
||||
|
||||
/**
|
||||
* The runtime's answer to a key it does not hold: the script was deployed under
|
||||
* a secret that has since rotated, and redeploying it binds the current key.
|
||||
*/
|
||||
const BAD_KEY_ERROR = 'bad-key';
|
||||
|
||||
/**
|
||||
* Runtime failure answers that are Puter's fault rather than the handler's: a
|
||||
* key the script no longer matches, no token to run as, or a handler that
|
||||
* failed because a Puter API call it made did.
|
||||
*/
|
||||
const PLATFORM_ERROR_CODES = new Set([
|
||||
BAD_KEY_ERROR,
|
||||
'no-token',
|
||||
'platform-error',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Set on the retry after this backend deploys a missing script itself, so the
|
||||
* dispatcher skips its own deploy-on-miss callback and negative cache and goes
|
||||
@@ -147,16 +95,10 @@ const SELF_DEPLOYABLE_MISS_REASONS = new Set([
|
||||
*/
|
||||
const ALREADY_DEPLOYED_MISS_REASONS = new Set(['deploy-timeout']);
|
||||
|
||||
/** Whose call asked for the deploy, for the share of deploys it may take. */
|
||||
export interface WorkerMissOptions {
|
||||
lane?: InvokeLane;
|
||||
}
|
||||
|
||||
/** What a self-deploy attempt did, and to which app/script it applies. */
|
||||
export type WorkerMissHandler = (
|
||||
appUid: string,
|
||||
script: string,
|
||||
options: WorkerMissOptions,
|
||||
) => Promise<'deployed' | string>;
|
||||
|
||||
/** How long a handler has to answer before the attempt is abandoned. */
|
||||
@@ -164,31 +106,12 @@ export const EVENTS_INVOKE_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Connections held open to the dispatcher, so a busy fleet is not
|
||||
* re-handshaking. Also how many calls one transport makes at once: a call past
|
||||
* that waits for a free one before its invoke timeout starts.
|
||||
* re-handshaking. The 30s invoke abort starts when a request is queued, not
|
||||
* when it gets a connection, so a saturated pool shows up as timeouts.
|
||||
*/
|
||||
export const INVOKE_CONNECTIONS = 256;
|
||||
|
||||
/**
|
||||
* Of those, how many schedule runs may hold at once, so a burst of schedules
|
||||
* can never take the connections event deliveries need.
|
||||
*/
|
||||
export const SCHEDULE_INVOKE_CONNECTIONS = 64;
|
||||
|
||||
/** Whose call this is, for the connection share it may take. */
|
||||
export type InvokeLane = 'delivery' | 'schedule';
|
||||
|
||||
/** What a deploy answers when another deploy of the region holds the way. */
|
||||
export const DEPLOY_BUSY = 'busy';
|
||||
const INVOKE_CONNECTIONS = 256;
|
||||
const INVOKE_KEEP_ALIVE_MS = 30_000;
|
||||
|
||||
/**
|
||||
* A call waits at most this share of its invoke timeout for a connection, so
|
||||
* the wait and the run together stay inside the worker lease (twice the
|
||||
* timeout). Past it the call fails as Puter's, not the handler's.
|
||||
*/
|
||||
const CONNECTION_WAIT_SHARE = 0.5;
|
||||
|
||||
/** One invocation, as the events service hands it over. */
|
||||
export interface WorkerInvokeRequest {
|
||||
/** Script the app's current handler set deploys as. */
|
||||
@@ -214,69 +137,36 @@ export interface EventsInvokeCall {
|
||||
timeoutMs: number;
|
||||
/** Set on the retry after this backend deployed the script itself. */
|
||||
deployed?: boolean;
|
||||
lane?: InvokeLane;
|
||||
/**
|
||||
* Builds `body` again at send time, for a transport that waits for a
|
||||
* connection first, so a send time stamped into it is the real one.
|
||||
*/
|
||||
render?: () => string;
|
||||
}
|
||||
|
||||
/** What one attempt at delivering an invocation came back with. */
|
||||
export interface EventsInvokeAnswer {
|
||||
/** `null` when no answer came back. */
|
||||
status: number | null;
|
||||
/** Set only when the answer provably came from the script. */
|
||||
handled?: boolean;
|
||||
error?: string;
|
||||
/** Set only by the dispatcher transport, naming its own failure. */
|
||||
dispatchReason?: string;
|
||||
/** No answer within the invoke timeout. */
|
||||
timedOut?: boolean;
|
||||
/** From a `retry-after` header in seconds. */
|
||||
retryAfterMs?: number;
|
||||
/** A deploy this call needed was put off: another held the way. */
|
||||
deployBusy?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delivers an invocation. `handled` is set only when the answer provably came
|
||||
* from the script, so an unmarked 4xx is not a handler's refusal.
|
||||
* Delivers an invocation. `status` null means nothing ran (retriable);
|
||||
* `handled` is set only when the answer provably came from the script, so an
|
||||
* unmarked 4xx is not a handler's refusal.
|
||||
*/
|
||||
export interface EventsInvokeTransport {
|
||||
send(call: EventsInvokeCall): Promise<EventsInvokeAnswer>;
|
||||
send(call: EventsInvokeCall): Promise<{
|
||||
status: number | null;
|
||||
handled?: boolean;
|
||||
error?: string;
|
||||
/** Set only by the dispatcher transport, naming its own failure. */
|
||||
dispatchReason?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* `settled`: handler accepted (2xx). `terminal`: handler refused (marked 4xx),
|
||||
* not retried. `retriable`: the handler failed or ran past the timeout.
|
||||
* `unavailable`: Puter could not start or finish the run (transport, dispatch
|
||||
* and deploy failures, unmarked answers, a Puter API failure inside the
|
||||
* handler); retried like `retriable` but never the handler's fault.
|
||||
* not retried. `retriable`: no handler answer (5xx, 429, timeout, transport
|
||||
* failure, unmarked 4xx).
|
||||
*/
|
||||
export type WorkerInvokeOutcome =
|
||||
| 'settled'
|
||||
| 'terminal'
|
||||
| 'retriable'
|
||||
| 'unavailable';
|
||||
export type WorkerInvokeOutcome = 'settled' | 'terminal' | 'retriable';
|
||||
|
||||
export interface WorkerInvokeResult {
|
||||
outcome: WorkerInvokeOutcome;
|
||||
/** A deploy on miss was put off because another one held the way. */
|
||||
deployBusy?: boolean;
|
||||
/** The status the handler answered with, or `null` when it never did. */
|
||||
status: number | null;
|
||||
/** Why it could not answer, when nothing did. */
|
||||
error?: string;
|
||||
/** How long the answer asked to be left alone. */
|
||||
retryAfterMs?: number;
|
||||
}
|
||||
|
||||
/** How one invocation is sent. */
|
||||
export interface WorkerInvokeOptions {
|
||||
lane?: InvokeLane;
|
||||
/** Applied to the event as each attempt is sent. */
|
||||
stampEvent?: (event: unknown) => unknown;
|
||||
}
|
||||
|
||||
const NO_TRANSPORT =
|
||||
@@ -297,8 +187,8 @@ export class EventsWorkerInvokerClient extends PuterClient {
|
||||
/**
|
||||
* Deploys a script the dispatcher reports missing, in place of the
|
||||
* rehydrate callback — for the case where that callback lands on a backend
|
||||
* other than this one — or one still bound to a rotated key. Set once;
|
||||
* `invoke()` calls it at most once per delivery and never loops.
|
||||
* other than this one. Set once; `invoke()` calls it at most once per
|
||||
* delivery and never loops.
|
||||
*/
|
||||
setMissHandler(handler: WorkerMissHandler): void {
|
||||
this.#missHandler = handler;
|
||||
@@ -309,92 +199,67 @@ export class EventsWorkerInvokerClient extends PuterClient {
|
||||
this.#dispatcher = null;
|
||||
}
|
||||
|
||||
async invoke(
|
||||
request: WorkerInvokeRequest,
|
||||
options: WorkerInvokeOptions = {},
|
||||
): Promise<WorkerInvokeResult> {
|
||||
async invoke(request: WorkerInvokeRequest): Promise<WorkerInvokeResult> {
|
||||
const transport = this.#pickTransport();
|
||||
if (!transport)
|
||||
return {
|
||||
outcome: 'unavailable',
|
||||
status: null,
|
||||
error: NO_TRANSPORT,
|
||||
};
|
||||
return { outcome: 'retriable', status: null, error: NO_TRANSPORT };
|
||||
|
||||
const { stampEvent } = options;
|
||||
const render = (): string =>
|
||||
JSON.stringify({
|
||||
const call: EventsInvokeCall = {
|
||||
script: request.script,
|
||||
appUid: request.appUid,
|
||||
key: request.key,
|
||||
body: JSON.stringify({
|
||||
handler: request.handler,
|
||||
event: stampEvent ? stampEvent(request.event) : request.event,
|
||||
event: request.event,
|
||||
ctx: request.ctx,
|
||||
token: request.token,
|
||||
});
|
||||
const send = (deployed: boolean): Promise<EventsInvokeAnswer> =>
|
||||
transport.send({
|
||||
script: request.script,
|
||||
appUid: request.appUid,
|
||||
key: request.key,
|
||||
body: render(),
|
||||
timeoutMs: this.#timeoutMs(),
|
||||
...(deployed ? { deployed: true } : {}),
|
||||
...(options.lane ? { lane: options.lane } : {}),
|
||||
...(stampEvent ? { render } : {}),
|
||||
});
|
||||
}),
|
||||
timeoutMs: this.#timeoutMs(),
|
||||
};
|
||||
|
||||
let result = await send(false);
|
||||
if (this.#missHandler && selfDeployable(result)) {
|
||||
const outcome = await this.#missHandler(
|
||||
request.appUid,
|
||||
request.script,
|
||||
{ lane: options.lane },
|
||||
);
|
||||
if (outcome !== 'deployed')
|
||||
return {
|
||||
outcome: 'unavailable',
|
||||
status: null,
|
||||
error: `deploy: ${outcome}`,
|
||||
...(outcome === DEPLOY_BUSY ? { deployBusy: true } : {}),
|
||||
};
|
||||
result = await send(true);
|
||||
} else if (
|
||||
result.status === null &&
|
||||
result.dispatchReason &&
|
||||
ALREADY_DEPLOYED_MISS_REASONS.has(result.dispatchReason)
|
||||
) {
|
||||
// No deploy to trigger — just give propagation the one retry
|
||||
// it usually needs, without spending an upload on it.
|
||||
result = await send(true);
|
||||
let result = await transport.send(call);
|
||||
if (result.status === null && result.dispatchReason) {
|
||||
if (
|
||||
this.#missHandler &&
|
||||
SELF_DEPLOYABLE_MISS_REASONS.has(result.dispatchReason)
|
||||
) {
|
||||
const outcome = await this.#missHandler(
|
||||
request.appUid,
|
||||
request.script,
|
||||
);
|
||||
if (outcome !== 'deployed')
|
||||
return {
|
||||
outcome: 'retriable',
|
||||
status: null,
|
||||
error: `deploy: ${outcome}`,
|
||||
};
|
||||
result = await transport.send({ ...call, deployed: true });
|
||||
} else if (
|
||||
ALREADY_DEPLOYED_MISS_REASONS.has(result.dispatchReason)
|
||||
) {
|
||||
// No deploy to trigger — just give propagation the one retry
|
||||
// it usually needs, without spending an upload on it.
|
||||
result = await transport.send({ ...call, deployed: true });
|
||||
}
|
||||
}
|
||||
|
||||
const { status, handled, error, timedOut, retryAfterMs } = result;
|
||||
if (result.deployBusy)
|
||||
return {
|
||||
outcome: 'unavailable',
|
||||
status: null,
|
||||
error,
|
||||
deployBusy: true,
|
||||
};
|
||||
// Past the timeout the handler was most likely still running, which
|
||||
// is its own doing; any other missing answer never reached it.
|
||||
const { status, handled, error } = result;
|
||||
if (status === null)
|
||||
return {
|
||||
outcome: timedOut ? 'retriable' : 'unavailable',
|
||||
status: null,
|
||||
error,
|
||||
};
|
||||
return { outcome: 'retriable', status: null, error };
|
||||
|
||||
const outcome = outcomeForStatus(status, handled === true, error);
|
||||
// An unmarked failure is not the handler's own answer; say so even
|
||||
// when the transport did not, so the reason a delivery kept retrying
|
||||
// shows up in the log rather than reading as an unexplained retry.
|
||||
const unmarked = status >= 300 && !handled;
|
||||
const outcome = outcomeForStatus(status, handled === true);
|
||||
// An unmarked 4xx (429 aside — that one is always retriable) is not
|
||||
// the handler's own answer; say so even when the transport did not,
|
||||
// so the reason a delivery kept retrying shows up in the log rather
|
||||
// than reading as an unexplained retry.
|
||||
const unmarkedFourxx =
|
||||
status !== 429 && status >= 400 && status < 500 && !handled;
|
||||
const reportedError =
|
||||
error ?? (unmarked ? `unmarked ${status}` : undefined);
|
||||
error ?? (unmarkedFourxx ? 'unmarked 4xx' : undefined);
|
||||
return {
|
||||
outcome,
|
||||
status,
|
||||
...(reportedError ? { error: reportedError } : {}),
|
||||
...(retryAfterMs ? { retryAfterMs } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -405,19 +270,9 @@ export class EventsWorkerInvokerClient extends PuterClient {
|
||||
// a misconfiguration worth reporting once per attempt rather than a
|
||||
// round trip.
|
||||
if (!dispatcherUrl || !internalSecret) return null;
|
||||
const { invokeBatchSize, invokeBatchWindowMs } =
|
||||
this.config.events ?? {};
|
||||
this.#dispatcher ??= new DispatcherInvokeTransport(
|
||||
dispatcherUrl,
|
||||
internalSecret,
|
||||
{
|
||||
...(invokeBatchSize !== undefined
|
||||
? { batchSize: invokeBatchSize }
|
||||
: {}),
|
||||
...(invokeBatchWindowMs !== undefined
|
||||
? { batchWindowMs: invokeBatchWindowMs }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
return this.#dispatcher;
|
||||
}
|
||||
@@ -430,132 +285,52 @@ export class EventsWorkerInvokerClient extends PuterClient {
|
||||
}
|
||||
}
|
||||
|
||||
/** A call waiting in, or sent with, the current batch. */
|
||||
interface BatchedCall {
|
||||
call: EventsInvokeCall;
|
||||
resolve: (answer: EventsInvokeAnswer) => void;
|
||||
/** When its wait for a connection began, for what is left of it. */
|
||||
waitingSince: number;
|
||||
}
|
||||
|
||||
export interface DispatcherInvokeTransportOptions {
|
||||
fetchImpl?: typeof undiciFetch;
|
||||
/** Calls per batch; 1 sends every call on its own. */
|
||||
batchSize?: number;
|
||||
batchWindowMs?: number;
|
||||
}
|
||||
|
||||
/** Reaches an events worker the only way production can: through the dispatcher. */
|
||||
export class DispatcherInvokeTransport implements EventsInvokeTransport {
|
||||
readonly #url: string;
|
||||
readonly #secret: string;
|
||||
readonly #fetchImpl: typeof undiciFetch;
|
||||
readonly #batchSize: number;
|
||||
readonly #batchWindowMs: number;
|
||||
#agent: Agent | null = null;
|
||||
/** Calls holding a connection slot. */
|
||||
#active = 0;
|
||||
/** Of those, schedule runs. */
|
||||
#scheduleActive = 0;
|
||||
/** Calls waiting for one, longest-waiting first, with their lane. */
|
||||
readonly #waiting = new Map<() => void, InvokeLane>();
|
||||
/** Calls gathered for the next batch. */
|
||||
#batch: BatchedCall[] = [];
|
||||
#batchTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
/** Until when the dispatcher is taken not to have the batch route. */
|
||||
#batchUnsupportedUntil = 0;
|
||||
|
||||
constructor(
|
||||
dispatcherUrl: string,
|
||||
secret: string,
|
||||
options: DispatcherInvokeTransportOptions = {},
|
||||
options: { fetchImpl?: typeof undiciFetch } = {},
|
||||
) {
|
||||
// Joined as a string: `new URL(path, base)` resolves an absolute path
|
||||
// against the base's origin, dropping any path prefix it carries.
|
||||
this.#url = dispatcherUrl.replace(/\/$/, '');
|
||||
this.#url = dispatcherUrl;
|
||||
this.#secret = secret;
|
||||
this.#fetchImpl = options.fetchImpl ?? undiciFetch;
|
||||
this.#batchSize = Math.max(
|
||||
1,
|
||||
Math.floor(options.batchSize ?? INVOKE_BATCH_SIZE),
|
||||
);
|
||||
this.#batchWindowMs = Math.max(
|
||||
0,
|
||||
options.batchWindowMs ?? INVOKE_BATCH_WINDOW_MS,
|
||||
);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.#flush();
|
||||
void this.#agent?.close().catch(() => {});
|
||||
this.#agent = null;
|
||||
}
|
||||
|
||||
async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
|
||||
// No `timedOut`: the handler never got this call, so the wait is not
|
||||
// its failure.
|
||||
const lane = call.lane ?? 'delivery';
|
||||
const waitingSince = Date.now();
|
||||
if (
|
||||
!(await this.#acquire(call.timeoutMs * CONNECTION_WAIT_SHARE, lane))
|
||||
)
|
||||
return {
|
||||
status: null,
|
||||
error: 'no free connection to the events dispatcher',
|
||||
};
|
||||
async send(call: EventsInvokeCall): Promise<{
|
||||
status: number | null;
|
||||
handled?: boolean;
|
||||
error?: string;
|
||||
dispatchReason?: string;
|
||||
}> {
|
||||
try {
|
||||
if (this.#batchSize === 1) return await this.#sendOne(call);
|
||||
return await new Promise<EventsInvokeAnswer>((resolve) => {
|
||||
this.#batch.push({ call, resolve, waitingSince });
|
||||
if (this.#batch.length >= this.#batchSize) this.#flush();
|
||||
else
|
||||
this.#batchTimer ??= setTimeout(
|
||||
() => this.#flush(),
|
||||
this.#batchWindowMs,
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
this.#release(lane);
|
||||
}
|
||||
}
|
||||
|
||||
/** Send what has gathered: alone when it is one call, else as a batch. */
|
||||
#flush(): void {
|
||||
if (this.#batchTimer) clearTimeout(this.#batchTimer);
|
||||
this.#batchTimer = null;
|
||||
const batch = this.#batch;
|
||||
this.#batch = [];
|
||||
if (batch.length === 0) return;
|
||||
if (batch.length === 1 || Date.now() < this.#batchUnsupportedUntil) {
|
||||
for (const { call, resolve } of batch)
|
||||
void this.#sendOne(call).then(resolve);
|
||||
return;
|
||||
}
|
||||
void this.#sendBatch(batch);
|
||||
}
|
||||
|
||||
/** One call on the single route. */
|
||||
async #sendOne(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
|
||||
try {
|
||||
const response = await this.#fetchImpl(
|
||||
`${this.#url}${EVENTS_DISPATCH_PATH}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-puter-internal-auth': this.#secret,
|
||||
'x-puter-events-script': call.script,
|
||||
'x-puter-events-app': call.appUid,
|
||||
'x-puter-events-key': call.key,
|
||||
...(call.deployed
|
||||
? { [EVENTS_DEPLOYED_HEADER]: '1' }
|
||||
: {}),
|
||||
},
|
||||
body: call.render ? call.render() : call.body,
|
||||
signal: AbortSignal.timeout(call.timeoutMs),
|
||||
dispatcher: this.#dispatcherFor(),
|
||||
// `new URL(path, base)` resolves an absolute path against the
|
||||
// base's origin, dropping any path prefix `base` already carries.
|
||||
const url = `${this.#url.replace(/\/$/, '')}${EVENTS_DISPATCH_PATH}`;
|
||||
const response = await this.#fetchImpl(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-puter-internal-auth': this.#secret,
|
||||
'x-puter-events-script': call.script,
|
||||
'x-puter-events-app': call.appUid,
|
||||
'x-puter-events-key': call.key,
|
||||
...(call.deployed ? { [EVENTS_DEPLOYED_HEADER]: '1' } : {}),
|
||||
},
|
||||
);
|
||||
body: call.body,
|
||||
signal: AbortSignal.timeout(call.timeoutMs),
|
||||
dispatcher: this.#dispatcherFor(),
|
||||
});
|
||||
// The body is not read: a handler answers with a status, and
|
||||
// whatever else it writes is its own business.
|
||||
void response.body?.cancel().catch(() => {});
|
||||
@@ -566,210 +341,24 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
|
||||
EVENTS_DISPATCH_ERROR_HEADER,
|
||||
);
|
||||
if (dispatchError !== null)
|
||||
return dispatchAnswer(dispatchError, response.status);
|
||||
return {
|
||||
status: null,
|
||||
error: `dispatcher: ${dispatchError} (${response.status})`,
|
||||
dispatchReason: dispatchError,
|
||||
};
|
||||
|
||||
return answerFrom(response);
|
||||
} catch (err) {
|
||||
return failedAnswer(err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Several calls in one request. A call may wait in the dispatcher's queue
|
||||
* for whatever is left of its connection wait, and its timeout runs from
|
||||
* its `started` line; the request itself is dropped once every call has an
|
||||
* answer.
|
||||
*/
|
||||
async #sendBatch(batch: BatchedCall[]): Promise<void> {
|
||||
const hangUp = new AbortController();
|
||||
let open = batch.length;
|
||||
const answered = batch.map(() => false);
|
||||
const timers: ReturnType<typeof setTimeout>[] = [];
|
||||
const settle = (i: number, answer: EventsInvokeAnswer): void => {
|
||||
if (answered[i]) return;
|
||||
answered[i] = true;
|
||||
clearTimeout(timers[i]);
|
||||
batch[i].resolve(answer);
|
||||
if (--open === 0) hangUp.abort();
|
||||
};
|
||||
const settleRest = (answer: EventsInvokeAnswer): void => {
|
||||
for (let i = 0; i < batch.length; i++) settle(i, answer);
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const waits = batch.map(({ call, waitingSince }) =>
|
||||
Math.max(
|
||||
0,
|
||||
Math.floor(
|
||||
call.timeoutMs * CONNECTION_WAIT_SHARE -
|
||||
(now - waitingSince),
|
||||
),
|
||||
),
|
||||
);
|
||||
const body = JSON.stringify({
|
||||
calls: batch.map(({ call }, i) => ({
|
||||
script: call.script,
|
||||
appUid: call.appUid,
|
||||
key: call.key,
|
||||
...(call.deployed ? { deployed: true } : {}),
|
||||
body: call.render ? call.render() : call.body,
|
||||
waitMs: waits[i],
|
||||
})),
|
||||
});
|
||||
const arm = (
|
||||
i: number,
|
||||
ms: number,
|
||||
answer: () => EventsInvokeAnswer,
|
||||
) => {
|
||||
clearTimeout(timers[i]);
|
||||
timers[i] = setTimeout(() => settle(i, answer()), ms);
|
||||
timers[i].unref?.();
|
||||
};
|
||||
// No `timedOut`: a call that never started never reached a handler.
|
||||
batch.forEach((_, i) =>
|
||||
arm(i, waits[i] + BATCH_START_GRACE_MS, () => ({
|
||||
status: null,
|
||||
error: 'dispatcher: call never started',
|
||||
})),
|
||||
);
|
||||
const started = (i: number): void => {
|
||||
if (!answered[i])
|
||||
arm(i, batch[i].call.timeoutMs, () =>
|
||||
failedAnswer(timeoutError()),
|
||||
);
|
||||
};
|
||||
const longest = Math.max(
|
||||
...batch.map(
|
||||
({ call }, i) =>
|
||||
waits[i] + BATCH_START_GRACE_MS + call.timeoutMs,
|
||||
),
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await this.#fetchImpl(
|
||||
`${this.#url}${EVENTS_DISPATCH_BATCH_PATH}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-puter-internal-auth': this.#secret,
|
||||
},
|
||||
body,
|
||||
signal: AbortSignal.any([
|
||||
hangUp.signal,
|
||||
AbortSignal.timeout(longest),
|
||||
]),
|
||||
dispatcher: this.#dispatcherFor(),
|
||||
},
|
||||
);
|
||||
|
||||
const dispatchError = response.headers.get(
|
||||
EVENTS_DISPATCH_ERROR_HEADER,
|
||||
);
|
||||
if (dispatchError === 'no-such-route') {
|
||||
// A dispatcher from before the batch route: nothing ran, so
|
||||
// every call goes again on its own.
|
||||
void response.body?.cancel().catch(() => {});
|
||||
this.#batchUnsupportedUntil =
|
||||
Date.now() + BATCH_UNSUPPORTED_RETRY_MS;
|
||||
batch.forEach(({ call }, i) => {
|
||||
if (!answered[i])
|
||||
void this.#sendOne(call).then((a) => settle(i, a));
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (dispatchError !== null || !response.ok) {
|
||||
void response.body?.cancel().catch(() => {});
|
||||
// An unmarked failure (an edge 404, a WAF 403) reads as
|
||||
// Puter's, never as any handler's answer.
|
||||
settleRest(
|
||||
dispatchError !== null
|
||||
? dispatchAnswer(dispatchError, response.status)
|
||||
: {
|
||||
status: null,
|
||||
error: `dispatcher: unmarked ${response.status} for a batch`,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let buffered = '';
|
||||
const read = (line: string): void => {
|
||||
if (line.trim() === '') return;
|
||||
const parsed = parseAnswerLine(line);
|
||||
if (!parsed || parsed.i >= batch.length) return;
|
||||
if ('started' in parsed) started(parsed.i);
|
||||
else settle(parsed.i, parsed.answer);
|
||||
const handled = response.headers.get(EVENTS_HANDLED_HEADER) === '1';
|
||||
const errorHeader = response.headers.get(EVENTS_ERROR_HEADER);
|
||||
return {
|
||||
status: response.status,
|
||||
handled,
|
||||
...(errorHeader ? { error: errorHeader } : {}),
|
||||
};
|
||||
const decoder = new TextDecoder();
|
||||
for await (const chunk of response.body ?? []) {
|
||||
buffered += decoder.decode(chunk, { stream: true });
|
||||
let newline = buffered.indexOf('\n');
|
||||
while (newline !== -1) {
|
||||
read(buffered.slice(0, newline));
|
||||
buffered = buffered.slice(newline + 1);
|
||||
newline = buffered.indexOf('\n');
|
||||
}
|
||||
}
|
||||
read(buffered);
|
||||
settleRest({
|
||||
status: null,
|
||||
error: 'dispatcher: batch ended without an answer',
|
||||
});
|
||||
} catch (err) {
|
||||
// Every call has its answer and the request was dropped on purpose.
|
||||
if (open === 0) return;
|
||||
settleRest(failedAnswer(err));
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a call on `lane` may take a connection now. */
|
||||
#free(lane: InvokeLane): boolean {
|
||||
return (
|
||||
this.#active < INVOKE_CONNECTIONS &&
|
||||
(lane !== 'schedule' ||
|
||||
this.#scheduleActive < SCHEDULE_INVOKE_CONNECTIONS)
|
||||
);
|
||||
}
|
||||
|
||||
#take(lane: InvokeLane): void {
|
||||
this.#active++;
|
||||
if (lane === 'schedule') this.#scheduleActive++;
|
||||
}
|
||||
|
||||
/** A connection slot, or false once `waitMs` passes without one. */
|
||||
#acquire(waitMs: number, lane: InvokeLane): Promise<boolean> {
|
||||
if (this.#free(lane)) {
|
||||
this.#take(lane);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const granted = () => {
|
||||
clearTimeout(timer);
|
||||
resolve(true);
|
||||
return {
|
||||
status: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
this.#waiting.delete(granted);
|
||||
resolve(false);
|
||||
}, waitMs);
|
||||
timer.unref?.();
|
||||
this.#waiting.set(granted, lane);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Give the slot back, then hand one to the longest waiter it can go to: a
|
||||
* schedule waiter at a full schedule share doesn't hold up a delivery.
|
||||
*/
|
||||
#release(lane: InvokeLane): void {
|
||||
this.#active--;
|
||||
if (lane === 'schedule') this.#scheduleActive--;
|
||||
for (const [granted, waiting] of this.#waiting) {
|
||||
if (!this.#free(waiting)) continue;
|
||||
this.#waiting.delete(granted);
|
||||
this.#take(waiting);
|
||||
granted();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -784,120 +373,18 @@ export class DispatcherInvokeTransport implements EventsInvokeTransport {
|
||||
}
|
||||
}
|
||||
|
||||
/** What a failure the dispatcher marked as its own reads as. */
|
||||
const dispatchAnswer = (
|
||||
reason: string,
|
||||
status: number,
|
||||
): EventsInvokeAnswer => ({
|
||||
status: null,
|
||||
error: `dispatcher: ${reason} (${status})`,
|
||||
dispatchReason: reason,
|
||||
});
|
||||
|
||||
/** What `AbortSignal.timeout` rejects with, for a batched call's own timeout. */
|
||||
const timeoutError = (): Error =>
|
||||
new DOMException(
|
||||
'The operation was aborted due to timeout',
|
||||
'TimeoutError',
|
||||
);
|
||||
|
||||
/** One line of a batch answer, or null when it is not one. */
|
||||
export const parseAnswerLine = (
|
||||
line: string,
|
||||
):
|
||||
| { i: number; started: true }
|
||||
| { i: number; answer: EventsInvokeAnswer }
|
||||
| null => {
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!raw || typeof raw !== 'object') return null;
|
||||
const { i, started, status, handled, error, retryAfter, dispatch } =
|
||||
raw as Record<string, unknown>;
|
||||
if (!Number.isInteger(i) || (i as number) < 0) return null;
|
||||
if (started === true) return { i: i as number, started: true };
|
||||
if (typeof status !== 'number') return null;
|
||||
if (typeof dispatch === 'string')
|
||||
return { i: i as number, answer: dispatchAnswer(dispatch, status) };
|
||||
return {
|
||||
i: i as number,
|
||||
answer: answerFrom({
|
||||
status,
|
||||
headers: {
|
||||
get: (name) => {
|
||||
if (name === EVENTS_HANDLED_HEADER)
|
||||
return handled === true ? '1' : null;
|
||||
if (name === EVENTS_ERROR_HEADER)
|
||||
return typeof error === 'string' ? error : null;
|
||||
if (name === 'retry-after')
|
||||
return typeof retryAfter === 'string'
|
||||
? retryAfter
|
||||
: null;
|
||||
return null;
|
||||
},
|
||||
},
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
/** Whether deploying the script is what fixes this answer. */
|
||||
const selfDeployable = (result: EventsInvokeAnswer): boolean =>
|
||||
result.status === null
|
||||
? result.dispatchReason !== undefined &&
|
||||
SELF_DEPLOYABLE_MISS_REASONS.has(result.dispatchReason)
|
||||
: result.handled === true && result.error === BAD_KEY_ERROR;
|
||||
|
||||
/** What a script's answer says, read the same way by every transport. */
|
||||
export const answerFrom = (response: {
|
||||
status: number;
|
||||
headers: { get(name: string): string | null };
|
||||
}): EventsInvokeAnswer => {
|
||||
const errorHeader = response.headers.get(EVENTS_ERROR_HEADER);
|
||||
const retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after'));
|
||||
return {
|
||||
status: response.status,
|
||||
handled: response.headers.get(EVENTS_HANDLED_HEADER) === '1',
|
||||
...(errorHeader ? { error: errorHeader } : {}),
|
||||
...(retryAfterMs ? { retryAfterMs } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
/** An attempt that threw before any answer came back. */
|
||||
export const failedAnswer = (err: unknown): EventsInvokeAnswer => ({
|
||||
status: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
// What `AbortSignal.timeout` rejects with.
|
||||
...((err as { name?: unknown } | null)?.name === 'TimeoutError'
|
||||
? { timedOut: true }
|
||||
: {}),
|
||||
});
|
||||
|
||||
/** Whole seconds only; an HTTP-date form reads as absent. */
|
||||
const parseRetryAfterMs = (value: string | null): number => {
|
||||
const seconds = Number(value);
|
||||
return value && Number.isFinite(seconds) && seconds > 0
|
||||
? seconds * 1000
|
||||
: 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* 2xx took it regardless. Anything else not marked as the script's own answer
|
||||
* came from something between here and the handler. A marked answer is a
|
||||
* refusal when it is a 4xx other than 429, Puter's failure when the runtime
|
||||
* names one, and the handler's failure otherwise.
|
||||
* 2xx took it regardless. 429 is always "not now". Any other 4xx is a refusal
|
||||
* only when `handled` says the answer is provably the script's own — an
|
||||
* unmarked 4xx came from something between here and the handler.
|
||||
*/
|
||||
export const outcomeForStatus = (
|
||||
status: number,
|
||||
handled: boolean,
|
||||
errorCode?: string,
|
||||
): WorkerInvokeOutcome => {
|
||||
if (status >= 200 && status < 300) return 'settled';
|
||||
if (!handled) return 'unavailable';
|
||||
if (errorCode && PLATFORM_ERROR_CODES.has(errorCode)) return 'unavailable';
|
||||
if (status === 429) return 'retriable';
|
||||
if (status >= 400 && status < 500) return 'terminal';
|
||||
if (status >= 400 && status < 500)
|
||||
return handled ? 'terminal' : 'retriable';
|
||||
return 'retriable';
|
||||
};
|
||||
@@ -1,85 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
describe('auth elevation cookies over HTTP', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
it.each(['bearer', 'cookie'])(
|
||||
'API host returns a usable token without setting cookies (%s auth)',
|
||||
async (credential) => {
|
||||
const { token, password, username } = env.users.user;
|
||||
const res = await fetch(new URL('/auth/elevate', env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(credential === 'cookie'
|
||||
? { cookie: `puter_auth_token=${token}` }
|
||||
: { authorization: `Bearer ${token}` }),
|
||||
},
|
||||
body: JSON.stringify({ password }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get('set-cookie')).toBeNull();
|
||||
const body = (await res.json()) as {
|
||||
elevated: boolean;
|
||||
token: string;
|
||||
};
|
||||
expect(body.elevated).toBe(true);
|
||||
expect(typeof body.token).toBe('string');
|
||||
const user = await env.server.stores.user.getByUsername(username);
|
||||
const { verifyStepUpSession } =
|
||||
await import('../../core/http/middleware/stepUpSession.js');
|
||||
expect(
|
||||
verifyStepUpSession(
|
||||
{
|
||||
headers: { 'x-puter-elevation': body.token },
|
||||
actor: { user: { uuid: user!.uuid } },
|
||||
} as never,
|
||||
{ tokenService: env.server.services.token },
|
||||
),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('GUI host sets the elevation cookie and omits the raw token for cookie auth', async () => {
|
||||
const { token, password } = env.users.user;
|
||||
const res = await fetch(new URL('/auth/elevate', env.origin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
cookie: `puter_auth_token=${token}`,
|
||||
},
|
||||
body: JSON.stringify({ password }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get('set-cookie')).toContain('puter_elevated=');
|
||||
expect(await res.json()).toEqual({ elevated: true });
|
||||
});
|
||||
});
|
||||
@@ -42,10 +42,6 @@ import {
|
||||
adminOnlyGate,
|
||||
requireUserActorGate,
|
||||
} from '../../core/http/middleware/gates.js';
|
||||
import {
|
||||
configureRateLimit,
|
||||
listConfiguredRateLimitBackends,
|
||||
} from '../../core/http/middleware/rateLimit.js';
|
||||
import {
|
||||
ROUTES_METADATA_KEY,
|
||||
type CollectedRoute,
|
||||
@@ -54,10 +50,7 @@ import {
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { createSecret as otpCreateSecret } from '../../services/auth/OTPUtil.js';
|
||||
import { kvSharePermission } from '../../services/kvShare/kvShares.js';
|
||||
import {
|
||||
FULL_API_ACCESS,
|
||||
NOTIFICATIONS_PUSH_PERMISSION,
|
||||
} from '../../services/permission/consts.js';
|
||||
import { FULL_API_ACCESS } from '../../services/permission/consts.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { resetCardVerificationStatusCache } from '../../util/cardFallback.js';
|
||||
import { FS_READ_LIMIT } from '../fs/limits.js';
|
||||
@@ -1757,40 +1750,6 @@ describe('AuthController.handleLoginOtp + handleLoginRecoveryCode', () => {
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('refuses the sign-in while the used-token store is down', async () => {
|
||||
const { user, secret, otpToken } = await startOtpLogin();
|
||||
const token = await otpToken();
|
||||
|
||||
const { default: backend } = listConfiguredRateLimitBackends();
|
||||
configureRateLimit({
|
||||
default: 'redis',
|
||||
redis: {
|
||||
multi: () => {
|
||||
throw new Error('store down');
|
||||
},
|
||||
},
|
||||
});
|
||||
const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
const res = makeRes();
|
||||
try {
|
||||
await expect(
|
||||
controller.handleLoginOtp(
|
||||
makeReq({ token, code: liveTotp(user.username, secret) }),
|
||||
res,
|
||||
),
|
||||
).rejects.toThrow('store down');
|
||||
} finally {
|
||||
logged.mockRestore();
|
||||
configureRateLimit({
|
||||
default: backend,
|
||||
redis: server.clients.redis,
|
||||
kv: server.stores.kv,
|
||||
metering: server.services.metering,
|
||||
});
|
||||
}
|
||||
expect(res.body).toBeUndefined();
|
||||
});
|
||||
|
||||
it('a wrong code leaves the token usable for a retry', async () => {
|
||||
const { user, secret, otpToken } = await startOtpLogin();
|
||||
const token = await otpToken();
|
||||
@@ -2220,78 +2179,6 @@ describe('AuthController grant flows', () => {
|
||||
).toContain(permission);
|
||||
});
|
||||
|
||||
it('grant-user-app: grants and revokes notifications:push for an app', async () => {
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `tgp-${uuidv4()}`,
|
||||
title: 'TestGrantPushApp',
|
||||
index_url: 'https://example.test/index.html',
|
||||
},
|
||||
{ ownerUserId: issuer.id },
|
||||
);
|
||||
const user = await server.stores.user.getById(issuer.id);
|
||||
const appActor = makeActor({
|
||||
user: user!,
|
||||
app: { uid: app.uid, id: app.id },
|
||||
});
|
||||
const hasPush = () =>
|
||||
server.services.permission.check(
|
||||
appActor,
|
||||
NOTIFICATIONS_PUSH_PERMISSION,
|
||||
);
|
||||
expect(await hasPush()).toBe(false);
|
||||
|
||||
// Anything past the bare permission names nothing a prompt describes.
|
||||
await expect(
|
||||
inCtx(issuerActor, () =>
|
||||
controller.handleGrantUserApp(
|
||||
makeReq(
|
||||
{
|
||||
app_uid: app.uid,
|
||||
permission: `${NOTIFICATIONS_PUSH_PERMISSION}:extra`,
|
||||
},
|
||||
{ actor: issuerActor },
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'permission_not_grantable',
|
||||
});
|
||||
expect(await hasPush()).toBe(false);
|
||||
|
||||
const res = makeRes();
|
||||
await inCtx(issuerActor, () =>
|
||||
controller.handleGrantUserApp(
|
||||
makeReq(
|
||||
{
|
||||
app_uid: app.uid,
|
||||
permission: NOTIFICATIONS_PUSH_PERMISSION,
|
||||
},
|
||||
{ actor: issuerActor },
|
||||
),
|
||||
res,
|
||||
),
|
||||
);
|
||||
expect(res.body).toEqual({});
|
||||
expect(await hasPush()).toBe(true);
|
||||
|
||||
await inCtx(issuerActor, () =>
|
||||
controller.handleRevokeUserApp(
|
||||
makeReq(
|
||||
{
|
||||
app_uid: app.uid,
|
||||
permission: NOTIFICATIONS_PUSH_PERMISSION,
|
||||
},
|
||||
{ actor: issuerActor },
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
);
|
||||
expect(await hasPush()).toBe(false);
|
||||
});
|
||||
|
||||
it('grant-user-app: resolves `origin` to the app when app_uid is omitted', async () => {
|
||||
// The GUI permission dialog and puter.perms.grantOrigin() identify
|
||||
// third-party sites by origin rather than app uid.
|
||||
@@ -5132,26 +5019,6 @@ describe('AuthController phone verification — staging & reuse', () => {
|
||||
expect(staged).toBe('+14155550123');
|
||||
});
|
||||
|
||||
it('makes an email-gate-last account verify its card before its phone', async () => {
|
||||
const { user, actor } = await makeUserAndActor({
|
||||
requires_phone_verification: 1,
|
||||
requires_card_verification: 1,
|
||||
});
|
||||
await server.stores.user.updateMetadata(user.id, {
|
||||
emailGateLast: true,
|
||||
});
|
||||
const prelude = stubPrelude();
|
||||
await withClients({ prelude }, async () => {
|
||||
await expect(
|
||||
controller.handleSendConfirmPhone(
|
||||
makeReq({ phone: '+14155550123' }, { actor }),
|
||||
makeRes(),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 409 });
|
||||
});
|
||||
expect(prelude.createVerification).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('forwards an abuse-extension veto as 429 with the opaque reason, and sends nothing', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const emitAndWait = vi.fn(
|
||||
@@ -5217,7 +5084,7 @@ describe('AuthController phone verification — staging & reuse', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('carries the client dispatch id on the check and sent events', async () => {
|
||||
it('carries the client dispatch id on the sent signal', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const emit = vi.fn();
|
||||
const emitAndWait = vi.fn(async () => {});
|
||||
@@ -5238,11 +5105,6 @@ describe('AuthController phone verification — staging & reuse', () => {
|
||||
expect.objectContaining({ dispatch_id: 'disp-123' }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(emitAndWait).toHaveBeenCalledWith(
|
||||
'puter.phone-verification.check',
|
||||
expect.objectContaining({ dispatch_id: 'disp-123' }),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('does NOT emit the sent signal when the send fails upstream', async () => {
|
||||
@@ -7750,46 +7612,6 @@ describe('AuthController permission revokes', () => {
|
||||
);
|
||||
expect(res.body).toEqual({});
|
||||
});
|
||||
|
||||
it('revoke-user-user: refuses a key-value share grant and leaves it standing', async () => {
|
||||
const { actor: ownerActor, user: owner } = await makeUserAndActor();
|
||||
const { user: holder } = await makeUserAndActor();
|
||||
const permissions = ['kv-share', 'kv-share-read', 'kv-share-write'].map(
|
||||
(root) => `${root}:${owner.uuid}:os-global:notes:${uuidv4()}`,
|
||||
);
|
||||
for (const permission of permissions)
|
||||
await server.stores.permission.upsertUserUserPerm(
|
||||
holder.id,
|
||||
owner.id,
|
||||
permission,
|
||||
{},
|
||||
);
|
||||
|
||||
for (const permission of permissions)
|
||||
await expect(
|
||||
inCtx(ownerActor, () =>
|
||||
controller.handleRevokeUserUser(
|
||||
makeReq(
|
||||
{ target_username: holder.username, permission },
|
||||
{ actor: ownerActor },
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
message: expect.stringContaining('puter.kv.unshare()'),
|
||||
});
|
||||
expect(
|
||||
(
|
||||
await server.stores.permission.findUserUserPermsByIssuer(
|
||||
holder.id,
|
||||
owner.id,
|
||||
permissions,
|
||||
)
|
||||
).sort(),
|
||||
).toEqual([...permissions].sort());
|
||||
});
|
||||
});
|
||||
|
||||
// ── Permission checks + listing ────────────────────────────────────
|
||||
|
||||
@@ -66,7 +66,6 @@ import {
|
||||
isCardFallbackEnabled,
|
||||
phoneAttemptsKey,
|
||||
} from '../../util/cardFallback.js';
|
||||
import { isEmailGateLast } from '../../util/verificationOrder.js';
|
||||
import { sessionCookieFlags } from '../../util/cookieFlags.js';
|
||||
import { cleanEmail, isStorableEmail } from '../../util/email.js';
|
||||
import { isGodmodeApp } from '../../util/godmodeApps.js';
|
||||
@@ -1221,18 +1220,6 @@ export class AuthController extends PuterController {
|
||||
}
|
||||
}
|
||||
|
||||
/** The email-gate-last confirmation email; a failure never fails the gate. */
|
||||
private async sendDeferredConfirmation(userId: number): Promise<void> {
|
||||
try {
|
||||
await this.services.signup.sendDeferredConfirmationEmail(userId);
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
'[verification] deferred confirmation email failed:',
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async isCardFallbackEligible(user: {
|
||||
id: number;
|
||||
requires_phone_verification?: boolean | number | null;
|
||||
@@ -1278,12 +1265,6 @@ export class AuthController extends PuterController {
|
||||
throw new HttpError(403, 'Account suspended.', {
|
||||
legacyCode: 'account_suspended',
|
||||
});
|
||||
if (isEmailGateLast(user) && user.requires_card_verification)
|
||||
throw new HttpError(
|
||||
409,
|
||||
'Card verification must be completed first.',
|
||||
{ legacyCode: 'conflict' },
|
||||
);
|
||||
if (!this.clients.prelude?.isConfigured())
|
||||
throw await this.smsSendError(
|
||||
503,
|
||||
@@ -1354,7 +1335,6 @@ export class AuthController extends PuterController {
|
||||
user_uid: user.uuid,
|
||||
phone: parsed.e164,
|
||||
device_fingerprint: req.deviceFingerprint ?? null,
|
||||
dispatch_id: dispatchId ?? null,
|
||||
allowed: true,
|
||||
reason: null as string | null,
|
||||
card_fallback_instead: false,
|
||||
@@ -1593,7 +1573,6 @@ export class AuthController extends PuterController {
|
||||
requires_phone_verification: 0,
|
||||
phone: pendingPhone,
|
||||
});
|
||||
await this.sendDeferredConfirmation(user.id);
|
||||
|
||||
// Run the verified-event listeners synchronously (emitAndWait, not
|
||||
// fire-and-forget emit) so a carrier-based card-verification waiver in
|
||||
@@ -1674,11 +1653,7 @@ export class AuthController extends PuterController {
|
||||
res.json({ card_verified: true });
|
||||
return;
|
||||
}
|
||||
if (
|
||||
user.requires_phone_verification &&
|
||||
!fallbackEligible &&
|
||||
!isEmailGateLast(user)
|
||||
)
|
||||
if (user.requires_phone_verification && !fallbackEligible)
|
||||
throw new HttpError(
|
||||
409,
|
||||
'Phone verification must be completed first.',
|
||||
@@ -1741,7 +1716,6 @@ export class AuthController extends PuterController {
|
||||
await this.stores.user.update(user.id, {
|
||||
requires_card_verification: 0,
|
||||
});
|
||||
await this.sendDeferredConfirmation(user.id);
|
||||
res.json({ card_verified: true, disabled: true });
|
||||
return;
|
||||
}
|
||||
@@ -1801,11 +1775,7 @@ export class AuthController extends PuterController {
|
||||
res.json({ card_verified: true });
|
||||
return;
|
||||
}
|
||||
if (
|
||||
user.requires_phone_verification &&
|
||||
!fallbackEligible &&
|
||||
!isEmailGateLast(user)
|
||||
)
|
||||
if (user.requires_phone_verification && !fallbackEligible)
|
||||
throw new HttpError(
|
||||
409,
|
||||
'Phone verification must be completed first.',
|
||||
@@ -1846,7 +1816,6 @@ export class AuthController extends PuterController {
|
||||
await this.stores.user.update(user.id, {
|
||||
requires_card_verification: 0,
|
||||
});
|
||||
await this.sendDeferredConfirmation(user.id);
|
||||
res.json({ card_verified: true, disabled: true });
|
||||
return;
|
||||
}
|
||||
@@ -1871,7 +1840,6 @@ export class AuthController extends PuterController {
|
||||
requires_card_verification: 0,
|
||||
...(clearedPhoneGate ? { requires_phone_verification: 0 } : {}),
|
||||
});
|
||||
await this.sendDeferredConfirmation(user.id);
|
||||
|
||||
// Awaited like `user.phone-verified`, so whatever a listener grants on
|
||||
// verification is in place before the client refreshes. emitAndWait
|
||||
@@ -3336,14 +3304,6 @@ export class AuthController extends PuterController {
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
// Only unsharing retires the handle along with its grant; withdrawing
|
||||
// the grant alone would leave the share listed and dead.
|
||||
if (typeof permission === 'string' && isKvSharePermission(permission))
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Stop a key-value share with `puter.kv.unshare()` (`DELETE /kv/shares/:handle`)',
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
await this.services.permission.revokeUserUserPermission(
|
||||
req.actor!,
|
||||
target_username,
|
||||
@@ -3850,9 +3810,8 @@ export class AuthController extends PuterController {
|
||||
|
||||
let app = await this.stores.app.getByUid(app_uid);
|
||||
if (!app && resolvedFromOrigin) {
|
||||
// Hosted-subdomain and custom-domain origins get the site owner
|
||||
// stamped as the app's creator at bootstrap; external origins
|
||||
// stay unowned.
|
||||
// Hosted-subdomain origins get the site owner stamped as the
|
||||
// app's creator at bootstrap; external origins stay unowned.
|
||||
// Canonical origin only, so alternate hosts share one row.
|
||||
const canonicalOrigin =
|
||||
this.services.auth.canonicalizeOrigin(origin);
|
||||
@@ -3868,10 +3827,9 @@ export class AuthController extends PuterController {
|
||||
// An origin's uid is a deterministic uuidv5, so a deleted app
|
||||
// reappears here under the identical uid. Withdraw any cross-app
|
||||
// data grants left pointing at it before this new row can inherit
|
||||
// consent the user gave its predecessor. Only `createFromOrigin`
|
||||
// reuses a uid, here or wherever else an origin app is bootstrapped:
|
||||
// `AppStore.create` mints a random uuid4, which no deleted app can
|
||||
// ever hold again.
|
||||
// consent the user gave its predecessor. Only *this* path can reuse
|
||||
// a uid: `AppStore.create` mints a random uuid4, which no deleted
|
||||
// app can ever hold again.
|
||||
//
|
||||
// Called directly rather than through `app.changed`: the token is
|
||||
// issued below, so this has to be able to stop that, and
|
||||
@@ -3882,9 +3840,6 @@ export class AuthController extends PuterController {
|
||||
app_uid,
|
||||
'uid reused by a new app',
|
||||
);
|
||||
// Key-value shares the predecessor left read as gone already,
|
||||
// being older than this row; this only cleans them up.
|
||||
this.services.kvShare.retireSharesOlderThanApp(app);
|
||||
}
|
||||
if (!app) {
|
||||
throw new HttpError(404, `App ${app_uid} does not exist`, {
|
||||
@@ -4320,27 +4275,7 @@ export class AuthController extends PuterController {
|
||||
);
|
||||
}
|
||||
|
||||
const code = req.body?.code;
|
||||
if (
|
||||
user.otp_enabled &&
|
||||
typeof code === 'string' &&
|
||||
/^[A-Z0-9]{8}$/.test(code)
|
||||
) {
|
||||
await this.#assertSecondFactorAttemptsRemain(user.uuid);
|
||||
const consumed = await this.stores.user.consumeOtpRecoveryCode(
|
||||
user.id,
|
||||
hashRecoveryCode(code),
|
||||
);
|
||||
if (!consumed) {
|
||||
await this.#recordSecondFactorFailure(user.uuid);
|
||||
throw new HttpError(401, 'Incorrect code.', {
|
||||
legacyCode: 'code_mismatch',
|
||||
fields: { factor: 'otp' },
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.#requireAccountTwoFactor(user, code);
|
||||
}
|
||||
this.#requireAccountTwoFactor(user, req.body?.code);
|
||||
|
||||
await this.stores.user.update(user.id, {
|
||||
otp_enabled: false,
|
||||
@@ -4550,14 +4485,11 @@ export class AuthController extends PuterController {
|
||||
}
|
||||
|
||||
const token = signStepUpToken(this.services.token, user as never);
|
||||
const isApiHost = req.subdomains?.at(-1) === 'api';
|
||||
if (!isApiHost) {
|
||||
res.cookie(
|
||||
STEP_UP_COOKIE_NAME,
|
||||
token,
|
||||
stepUpCookieOptions(this.config),
|
||||
);
|
||||
}
|
||||
res.cookie(
|
||||
STEP_UP_COOKIE_NAME,
|
||||
token,
|
||||
stepUpCookieOptions(this.config),
|
||||
);
|
||||
|
||||
// A browser reads its elevation back from the httpOnly cookie and never
|
||||
// needs the raw value; handing it to page JS would put the second factor
|
||||
@@ -4566,9 +4498,7 @@ export class AuthController extends PuterController {
|
||||
// password/TOTP — this only avoids needless exposure, it isn't a gate.
|
||||
const cookieName = this.config.cookie_name ?? 'puter_token';
|
||||
const usedSessionCookie =
|
||||
!isApiHost &&
|
||||
!!req.token &&
|
||||
req.token === req.cookies?.[cookieName];
|
||||
!!req.token && req.token === req.cookies?.[cookieName];
|
||||
res.json(
|
||||
usedSessionCookie ? { elevated: true } : { elevated: true, token },
|
||||
);
|
||||
@@ -4681,19 +4611,13 @@ export class AuthController extends PuterController {
|
||||
await checkRateLimit(`${scope}:${userUid}`, limit, window);
|
||||
}
|
||||
|
||||
/**
|
||||
* An `otp-login` JWT completes one sign-in; a second redemption is refused.
|
||||
* Fails closed: while the store can't say whether the token was used, the
|
||||
* sign-in is refused rather than letting a captured token replay.
|
||||
*/
|
||||
/** An `otp-login` JWT completes one sign-in; a second redemption is refused. */
|
||||
async #consumeOtpLoginToken(token: string): Promise<void> {
|
||||
const digest = crypto.createHash('sha256').update(token).digest('hex');
|
||||
const firstUse = await checkRateLimit(
|
||||
`login-otp-token-used:${digest}`,
|
||||
1,
|
||||
OTP_LOGIN_TOKEN_USED_WINDOW_MS,
|
||||
undefined,
|
||||
{ failOpen: false },
|
||||
);
|
||||
if (!firstUse) {
|
||||
throw new HttpError(400, 'Invalid token.', {
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { hashRecoveryCode } from '../../services/auth/OTPUtil.js';
|
||||
import {
|
||||
createTestUser,
|
||||
setupPuterTestEnv,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
|
||||
describe('2FA after signing in with a recovery code', () => {
|
||||
let env: PuterTestEnv;
|
||||
const password = 'lost-authenticator-password';
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = (
|
||||
path: string,
|
||||
body: Record<string, unknown>,
|
||||
cookie?: string,
|
||||
) =>
|
||||
fetch(new URL(path, env.origin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(cookie ? { cookie } : {}),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
it('allows an unused recovery code to disable 2FA after password revalidation', async () => {
|
||||
const username = `lost2fa_${uuidv4().slice(0, 8)}`;
|
||||
await createTestUser(env.server, { username, password });
|
||||
const user = (await env.server.stores.user.getByUsername(username))!;
|
||||
await env.server.stores.user.update(user.id, {
|
||||
email: `${username}@example.com`,
|
||||
email_confirmed: true,
|
||||
otp_enabled: true,
|
||||
otp_secret: 'TESTSECRETBASE32',
|
||||
otp_recovery_codes: ['RECOVER1', 'RECOVER2', 'RECOVER3']
|
||||
.map(hashRecoveryCode)
|
||||
.join(','),
|
||||
});
|
||||
|
||||
// Sign in with a recovery code: works, and yields a web session.
|
||||
const login = await post('/login', { username, password });
|
||||
expect(login.status).toBe(202);
|
||||
const { otp_jwt_token } = await login.json();
|
||||
const recovered = await post('/login/recovery-code', {
|
||||
token: otp_jwt_token,
|
||||
code: 'RECOVER1',
|
||||
});
|
||||
expect((await recovered.json()).next_step).toBe('complete');
|
||||
const cookie = recovered.headers
|
||||
.getSetCookie()
|
||||
.find((value) => value.startsWith('puter_auth_token='))!
|
||||
.split(';')[0]!;
|
||||
|
||||
const reused = await post(
|
||||
'/user-protected/disable-2fa',
|
||||
{ password, code: 'RECOVER1' },
|
||||
cookie,
|
||||
);
|
||||
expect(reused.status).toBe(401);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, { force: true }))!
|
||||
.otp_enabled,
|
||||
).toBeTruthy();
|
||||
|
||||
const disable = await post(
|
||||
'/user-protected/disable-2fa',
|
||||
{ password, code: 'RECOVER2' },
|
||||
cookie,
|
||||
);
|
||||
expect(disable.status).toBe(200);
|
||||
expect(await disable.json()).toMatchObject({ success: true });
|
||||
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.otp_enabled).toBeFalsy();
|
||||
expect(after!.otp_secret).toBeNull();
|
||||
expect(after!.otp_recovery_codes).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -19,7 +19,6 @@
|
||||
|
||||
import type { Request, Response } from 'express';
|
||||
import { Controller, Post } from '../../core/http/decorators.js';
|
||||
import { schedulesUnavailable } from '../../core/schedules.js';
|
||||
import type { BroadcastService } from '../../services/broadcast/BroadcastService.js';
|
||||
import type { ForwardBatch } from '../../services/events/forwardQueue.js';
|
||||
import { PuterController } from '../types.js';
|
||||
@@ -109,42 +108,6 @@ export class BroadcastController extends PuterController {
|
||||
);
|
||||
res.status(200).json({ ok: true, ...reply });
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule changes a region made for schedules this one owns, handed to
|
||||
* `services.schedules`; `events_disabled` on a server without it.
|
||||
*/
|
||||
@Post('/events/schedules', { subdomain: '*' })
|
||||
async schedules(req: Request, res: Response): Promise<void> {
|
||||
const broadcast = this.services.broadcast as unknown as
|
||||
BroadcastService | undefined;
|
||||
if (!broadcast) {
|
||||
res.status(503).json({
|
||||
error: { message: 'Broadcast service not registered' },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const verified = await broadcast.verifySignedRequest(
|
||||
req.rawBody,
|
||||
broadcastHeaders(req),
|
||||
);
|
||||
if (!verified.ok) {
|
||||
res.status(verified.status ?? 400).json({
|
||||
error: { message: verified.message ?? 'Bad request' },
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (verified.info?.ignored) {
|
||||
res.status(200).json({ ok: true, ...verified.info });
|
||||
return;
|
||||
}
|
||||
|
||||
const schedules = this.services.schedules;
|
||||
if (!schedules) throw schedulesUnavailable();
|
||||
const reply = await schedules.receivePeerChanges(req.body);
|
||||
res.status(200).json({ ok: true, ...reply });
|
||||
}
|
||||
}
|
||||
|
||||
const broadcastHeaders = (req: Request) => {
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import type { Request, RequestHandler, Response } from 'express';
|
||||
import { beforeEach, describe, expect, it } from 'vitest';
|
||||
import { Context, runWithContext } from '../../core/context.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { isHttpError } from '../../core/http/HttpError.js';
|
||||
import { configureRateLimit } from '../../core/http/middleware/rateLimit.js';
|
||||
import { DriverController } from './DriverController.js';
|
||||
@@ -278,86 +278,6 @@ describe('DriverController — concurrent acquire/release', () => {
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
// A driver that keeps working after the caller hangs up (async video
|
||||
// polling) must keep counting against the cap until it settles.
|
||||
it('holds the slot while the driver still works after a disconnect', async () => {
|
||||
let finish = () => {};
|
||||
const driver = {
|
||||
...makeSyntheticDriver(),
|
||||
ping: () =>
|
||||
new Promise<string>((resolve) => {
|
||||
finish = () => resolve('pong');
|
||||
}),
|
||||
};
|
||||
const handler = captureCallHandler(buildController(driver));
|
||||
const req = () =>
|
||||
makeReq({ interface: 'test-iface', method: 'ping' }, 'working');
|
||||
|
||||
const res1 = new StubRes();
|
||||
const first = callInScope(handler, req(), res1 as unknown as Response);
|
||||
await new Promise((r) => setImmediate(r));
|
||||
res1.emit('close');
|
||||
await new Promise((r) => setImmediate(r));
|
||||
|
||||
await expect(
|
||||
callInScope(handler, req(), new StubRes() as unknown as Response),
|
||||
).rejects.toSatisfy(
|
||||
(e) =>
|
||||
isHttpError(e) &&
|
||||
(e as { statusCode: number }).statusCode === 429,
|
||||
);
|
||||
|
||||
finish();
|
||||
await first;
|
||||
await new Promise((r) => setImmediate(r));
|
||||
const res2 = new StubRes();
|
||||
const second = callInScope(handler, req(), res2 as unknown as Response);
|
||||
await new Promise((r) => setImmediate(r));
|
||||
finish();
|
||||
await second;
|
||||
res2.emit('finish');
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
it('holds the slot for work a driver keeps past its response', async () => {
|
||||
let settleWork = () => {};
|
||||
const driver = {
|
||||
...makeSyntheticDriver(),
|
||||
ping: async () => {
|
||||
Context.get('keepDriverSlot')?.(
|
||||
new Promise<void>((resolve) => {
|
||||
settleWork = resolve;
|
||||
}),
|
||||
);
|
||||
return 'pong';
|
||||
},
|
||||
};
|
||||
const handler = captureCallHandler(buildController(driver));
|
||||
const req = () =>
|
||||
makeReq({ interface: 'test-iface', method: 'ping' }, 'kept');
|
||||
|
||||
const res1 = new StubRes();
|
||||
await callInScope(handler, req(), res1 as unknown as Response);
|
||||
res1.emit('finish');
|
||||
await new Promise((r) => setImmediate(r));
|
||||
|
||||
await expect(
|
||||
callInScope(handler, req(), new StubRes() as unknown as Response),
|
||||
).rejects.toSatisfy(
|
||||
(e) =>
|
||||
isHttpError(e) &&
|
||||
(e as { statusCode: number }).statusCode === 429,
|
||||
);
|
||||
|
||||
settleWork();
|
||||
await new Promise((r) => setImmediate(r));
|
||||
const res2 = new StubRes();
|
||||
await callInScope(handler, req(), res2 as unknown as Response);
|
||||
res2.emit('finish');
|
||||
settleWork();
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
it('does not attach release listeners when the driver declares no concurrent config', async () => {
|
||||
// With no spec there is nothing to release, so the gate must not
|
||||
// wire `finish`/`close` release listeners. The client-disconnect
|
||||
|
||||
@@ -736,42 +736,6 @@ describe('DriverController driver-method lifecycle events', () => {
|
||||
rejectReason: 'blocked in test',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers 429 when a before listener vetoes with rejectStatus 429', async () => {
|
||||
const actor = await makeUserActor();
|
||||
await grantKvPerm(actor);
|
||||
const key = `throttle-${uuidv4()}`;
|
||||
server.clients.event.on(
|
||||
'driver.puter-kvstore.set.before',
|
||||
(_k, data) => {
|
||||
if ((data.args as { key?: string })?.key === key) {
|
||||
data.allow = false;
|
||||
data.rejectStatus = 429;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const req = makeReq(
|
||||
{
|
||||
interface: 'puter-kvstore',
|
||||
method: 'set',
|
||||
args: { key, value: 'v' },
|
||||
},
|
||||
actor,
|
||||
);
|
||||
await expect(
|
||||
runWithContext({ actor }, () =>
|
||||
routes['POST /call'](
|
||||
req,
|
||||
makeRes() as unknown as Response,
|
||||
() => {},
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 429,
|
||||
legacyCode: 'too_many_requests',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriverController.#handleListInterfaces', () => {
|
||||
|
||||
@@ -31,7 +31,6 @@ import {
|
||||
checkDriverRateLimit,
|
||||
} from '../../core/http/middleware/rateLimit.js';
|
||||
import type { PuterRouter } from '../../core/http/PuterRouter.js';
|
||||
import type { LifecycleRejectStatus } from '../../clients/event/types.js';
|
||||
import type { DriverMeta } from '../../drivers/meta.js';
|
||||
import {
|
||||
isCreditExhaustion,
|
||||
@@ -51,7 +50,6 @@ import { PermissionUtil } from '../../services/permission/permissionUtil.js';
|
||||
import type { WithLifecycle } from '../../types';
|
||||
import { withSpan } from '../../util/span.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import { CallSlot } from './callSlot.js';
|
||||
|
||||
type DriverInstance = WithLifecycle & Record<string, unknown>;
|
||||
|
||||
@@ -261,17 +259,6 @@ export class DriverController extends PuterController {
|
||||
// -- Handlers ----------------------------------------------------
|
||||
|
||||
#handleCall = async (req: Request, res: Response): Promise<void> => {
|
||||
const slot = new CallSlot();
|
||||
const call = this.#call(req, res, slot);
|
||||
slot.keep(call);
|
||||
await call;
|
||||
};
|
||||
|
||||
#call = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
slot: CallSlot,
|
||||
): Promise<void> => {
|
||||
const {
|
||||
interface: ifaceName,
|
||||
method,
|
||||
@@ -335,8 +322,7 @@ export class DriverController extends PuterController {
|
||||
|
||||
if (req.actor) {
|
||||
const permService = this.services.permission as unknown as
|
||||
| PermissionService
|
||||
| undefined;
|
||||
PermissionService | undefined;
|
||||
if (permService) {
|
||||
// Build via PermissionUtil.join so any `:` in a driver or
|
||||
// interface name is escaped — raw interpolation would let a
|
||||
@@ -409,9 +395,10 @@ export class DriverController extends PuterController {
|
||||
|
||||
// Per-method rate-limit and concurrent specs both live on the
|
||||
// driver's resolved meta. Rate-limit is single-shot; concurrent
|
||||
// acquires a slot held until the response is done and the call's
|
||||
// work has settled, so streamed responses hold it until the stream
|
||||
// drains and work that continues after a disconnect still counts.
|
||||
// acquires a slot that must be released when the response is done
|
||||
// — we hook `res.finish` / `res.close` for that so streamed
|
||||
// responses hold their slot until the stream drains, and aborted
|
||||
// requests still give the slot back.
|
||||
const rateLimitSpec = resolvePerMethod(driverMeta?.rateLimit, method);
|
||||
if (
|
||||
!(await checkDriverRateLimit(req, ifaceName, method, rateLimitSpec))
|
||||
@@ -443,8 +430,18 @@ export class DriverController extends PuterController {
|
||||
legacyCode: 'too_many_requests',
|
||||
});
|
||||
}
|
||||
slot.attach(handle, res);
|
||||
Context.set('keepDriverSlot', (work) => slot.keep(work));
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
void handle.release();
|
||||
};
|
||||
// If the handler throws before responding, the express error
|
||||
// handler will eventually send a response — `finish` fires then,
|
||||
// so we still release. `close` covers client aborts.
|
||||
res.once('finish', release);
|
||||
res.once('close', release);
|
||||
if (res.destroyed || res.writableFinished) release();
|
||||
}
|
||||
|
||||
// Stash the requested driver name in Context so multi-provider
|
||||
@@ -477,7 +474,6 @@ export class DriverController extends PuterController {
|
||||
args,
|
||||
allow: true as boolean,
|
||||
rejectReason: undefined as string | undefined,
|
||||
rejectStatus: undefined as LifecycleRejectStatus | undefined,
|
||||
};
|
||||
await this.clients.event?.emitAndWait(
|
||||
`driver.${ifaceName}.${method}.before`,
|
||||
@@ -499,12 +495,11 @@ export class DriverController extends PuterController {
|
||||
},
|
||||
{},
|
||||
);
|
||||
const throttled = beforeEvent.rejectStatus === 429;
|
||||
throw new HttpError(
|
||||
throttled ? 429 : 403,
|
||||
403,
|
||||
beforeEvent.rejectReason ??
|
||||
`Blocked by policy: ${ifaceName}:${method}`,
|
||||
{ legacyCode: throttled ? 'too_many_requests' : 'forbidden' },
|
||||
{ legacyCode: 'forbidden' },
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'node:events';
|
||||
import type { Response } from 'express';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { CallSlot } from './callSlot.js';
|
||||
|
||||
const handle = () => ({
|
||||
release: vi.fn(() => Promise.resolve()),
|
||||
renew: vi.fn(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const deferred = () => {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((r) => {
|
||||
resolve = r;
|
||||
});
|
||||
return { promise, resolve };
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('CallSlot', () => {
|
||||
it('renews a slot held by a long call and stops once it is released', async () => {
|
||||
vi.useFakeTimers();
|
||||
const slot = new CallSlot();
|
||||
const slotHandle = handle();
|
||||
const res = new EventEmitter();
|
||||
const call = deferred();
|
||||
slot.keep(call.promise);
|
||||
slot.attach(slotHandle, res as unknown as Response);
|
||||
|
||||
res.emit('close');
|
||||
await vi.advanceTimersByTimeAsync(25 * 60 * 1000);
|
||||
expect(slotHandle.renew).toHaveBeenCalledTimes(2);
|
||||
expect(slotHandle.release).not.toHaveBeenCalled();
|
||||
|
||||
call.resolve();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(slotHandle.release).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(25 * 60 * 1000);
|
||||
expect(slotHandle.renew).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('releases once the response is done if no work is outstanding', async () => {
|
||||
const slot = new CallSlot();
|
||||
const slotHandle = handle();
|
||||
const res = new EventEmitter();
|
||||
slot.attach(slotHandle, res as unknown as Response);
|
||||
res.emit('finish');
|
||||
expect(slotHandle.release).toHaveBeenCalledOnce();
|
||||
expect(slotHandle.renew).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,80 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { Response } from 'express';
|
||||
|
||||
// Under the concurrency store's one-hour orphan TTL.
|
||||
const SLOT_RENEW_MS = 10 * 60 * 1000;
|
||||
|
||||
interface SlotHandle {
|
||||
release(): Promise<void>;
|
||||
renew?(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A driver call's concurrency slot, held until the response is done and the
|
||||
* call's work has settled, including work a driver carries on after the caller
|
||||
* hung up or was answered.
|
||||
*/
|
||||
export class CallSlot {
|
||||
#handle?: SlotHandle;
|
||||
#work = new Set<Promise<unknown>>();
|
||||
#responded = false;
|
||||
#released = false;
|
||||
#renew?: NodeJS.Timeout;
|
||||
|
||||
attach(handle: SlotHandle, res: Response): void {
|
||||
this.#handle = handle;
|
||||
const responded = () => {
|
||||
this.#responded = true;
|
||||
this.#maybeRelease();
|
||||
};
|
||||
res.once('finish', responded);
|
||||
res.once('close', responded);
|
||||
if (res.destroyed || res.writableFinished) responded();
|
||||
if (!this.#released && this.#work.size > 0) this.#startRenewing();
|
||||
}
|
||||
|
||||
keep(work: Promise<unknown>): void {
|
||||
if (this.#released) return;
|
||||
this.#work.add(work);
|
||||
if (this.#handle) this.#startRenewing();
|
||||
const settled = () => {
|
||||
this.#work.delete(work);
|
||||
this.#maybeRelease();
|
||||
};
|
||||
work.then(settled, settled);
|
||||
}
|
||||
|
||||
#startRenewing(): void {
|
||||
if (this.#renew) return;
|
||||
this.#renew = setInterval(() => {
|
||||
void this.#handle?.renew?.();
|
||||
}, SLOT_RENEW_MS);
|
||||
this.#renew.unref?.();
|
||||
}
|
||||
|
||||
#maybeRelease(): void {
|
||||
if (this.#released || !this.#handle) return;
|
||||
if (!this.#responded || this.#work.size > 0) return;
|
||||
this.#released = true;
|
||||
clearInterval(this.#renew);
|
||||
void this.#handle.release();
|
||||
}
|
||||
}
|
||||
@@ -25,11 +25,6 @@ import { bodyRecord } from '../../core/http/requestBody.js';
|
||||
import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js';
|
||||
import { EVENTS_WORKERS_LIST_LIMIT_CAP } from '../../stores/events/EventHandlerStore.js';
|
||||
import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
|
||||
import {
|
||||
SCHEDULE_LIST_LIMIT_CAP,
|
||||
schedulesUnavailable,
|
||||
type ScheduleServiceContract,
|
||||
} from '../../core/schedules.js';
|
||||
import { normalizeLimit } from '../../util/pagination.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import { EVENTS_WORKER_PREFIX } from '../../services/events/workerRuntime.js';
|
||||
@@ -109,24 +104,6 @@ export class EventsController extends PuterController {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /events/subscriptions/update — replace one subscription's `context`
|
||||
* in place. An id the caller does not hold reads as absent.
|
||||
*/
|
||||
@Post('/subscriptions/update', {
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
allowAccessToken: true,
|
||||
})
|
||||
async update(req: Request, res: Response): Promise<void> {
|
||||
const actor = this.#requireActor(req);
|
||||
const { sub } = await this.services.events.updateDurable(
|
||||
actor,
|
||||
bodyRecord(req),
|
||||
);
|
||||
res.json(sub);
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /events/fetch — what the caller missed, one page at a time.
|
||||
*
|
||||
@@ -174,88 +151,6 @@ export class EventsController extends PuterController {
|
||||
res.json({});
|
||||
}
|
||||
|
||||
// -- Schedules ---------------------------------------------------
|
||||
//
|
||||
// An app's handler run on a timer, served by `services.schedules` when a
|
||||
// server has it and answered `events_disabled` when it doesn't. Creating
|
||||
// one needs an app context; listing, reading and removing are scoped the
|
||||
// way the durable verbs are. The create and remove verbs spend the
|
||||
// subscribe and unsubscribe budgets in the service, shared with those
|
||||
// verbs.
|
||||
|
||||
#schedules(): ScheduleServiceContract {
|
||||
const schedules = this.services.schedules;
|
||||
if (!schedules) throw schedulesUnavailable();
|
||||
return schedules;
|
||||
}
|
||||
|
||||
/** POST /events/schedules — create a schedule for the calling app. */
|
||||
@Post('/schedules', {
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
allowAccessToken: true,
|
||||
})
|
||||
async createSchedule(req: Request, res: Response): Promise<void> {
|
||||
const actor = this.#requireActor(req);
|
||||
res.json(await this.#schedules().createFor(actor, bodyRecord(req)));
|
||||
}
|
||||
|
||||
/** GET /events/schedules — the caller's schedules, one page at a time. */
|
||||
@Get('/schedules', {
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
allowAccessToken: true,
|
||||
rateLimit: EVENTS_LIST_LIMIT,
|
||||
})
|
||||
async listSchedules(req: Request, res: Response): Promise<void> {
|
||||
const actor = this.#requireActor(req);
|
||||
const query = (req.query ?? {}) as Record<string, unknown>;
|
||||
|
||||
const page = await this.#schedules().list(actor, {
|
||||
limit: normalizeLimit(query.limit, {
|
||||
cap: SCHEDULE_LIST_LIMIT_CAP,
|
||||
}),
|
||||
cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
|
||||
includeTotal: query.includeTotal === 'true',
|
||||
});
|
||||
|
||||
res.json({
|
||||
items: page.items,
|
||||
...(page.cursor ? { cursor: page.cursor } : {}),
|
||||
...(page.total !== undefined ? { total: page.total } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /events/schedules/remove — end one schedule. An id the caller does
|
||||
* not hold reads as absent.
|
||||
*/
|
||||
@Post('/schedules/remove', {
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
allowAccessToken: true,
|
||||
})
|
||||
async removeSchedule(req: Request, res: Response): Promise<void> {
|
||||
const actor = this.#requireActor(req);
|
||||
await this.#schedules().remove(actor, bodyRecord(req).scheduleId);
|
||||
res.json({});
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /events/schedules/:scheduleId — one schedule, ended or not. An id the
|
||||
* caller does not hold reads as absent.
|
||||
*/
|
||||
@Get('/schedules/:scheduleId', {
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
allowAccessToken: true,
|
||||
rateLimit: EVENTS_LIST_LIMIT,
|
||||
})
|
||||
async getSchedule(req: Request, res: Response): Promise<void> {
|
||||
const actor = this.#requireActor(req);
|
||||
res.json(await this.#schedules().get(actor, req.params.scheduleId));
|
||||
}
|
||||
|
||||
// -- Cross-user key-value handles --------------------------------
|
||||
|
||||
/**
|
||||
@@ -512,14 +407,10 @@ export class EventsController extends PuterController {
|
||||
res.json({ deployed: true });
|
||||
return;
|
||||
}
|
||||
// `failed`, `throttled` and `busy` are worth another attempt: the rest
|
||||
// say this script is not what the app's handlers currently are.
|
||||
// `failed` and `throttled` are worth another attempt: the rest say
|
||||
// this script is not what the app's handlers currently are.
|
||||
res.status(
|
||||
outcome === 'failed' ||
|
||||
outcome === 'throttled' ||
|
||||
outcome === 'busy'
|
||||
? 502
|
||||
: 404,
|
||||
outcome === 'failed' || outcome === 'throttled' ? 502 : 404,
|
||||
).json({
|
||||
deployed: false,
|
||||
reason: outcome,
|
||||
@@ -546,8 +437,8 @@ export class EventsController extends PuterController {
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.services.events.useWorkerMissHandler((appUid, script, options) =>
|
||||
deployer.ensure(appUid, script, options),
|
||||
this.services.events.useWorkerMissHandler((appUid, script) =>
|
||||
deployer.ensure(appUid, script),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -41,30 +41,16 @@ import {
|
||||
EVENTS_SINGLE_DELIVERY_LIMIT,
|
||||
EVENTS_WORKER_INVOCATION_LIMIT,
|
||||
limitFor,
|
||||
tiered,
|
||||
type TieredLimit,
|
||||
} from './limits.js';
|
||||
|
||||
const tiers: Array<[string, TieredLimit]> = [
|
||||
[
|
||||
'durable subscriptions per account',
|
||||
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER,
|
||||
],
|
||||
['durable subscriptions per account', EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER],
|
||||
['durable subscriptions per app', EVENTS_DURABLE_SUBSCRIPTIONS_PER_APP],
|
||||
['live share handles per account', EVENTS_KV_HANDLES_PER_USER],
|
||||
['live share handles per app', EVENTS_KV_HANDLES_PER_APP],
|
||||
];
|
||||
|
||||
describe('tiered', () => {
|
||||
it('gives paid plans the base, and the free and temporary tiers their own', () => {
|
||||
const limit = tiered(30, 20, 10);
|
||||
expect(limitFor(limit, null)).toBe(30);
|
||||
expect(limitFor(limit, 'some_plan')).toBe(30);
|
||||
expect(limitFor(limit, DEFAULT_FREE_SUBSCRIPTION)).toBe(20);
|
||||
expect(limitFor(limit, DEFAULT_TEMP_SUBSCRIPTION)).toBe(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the tiered subscription quotas', () => {
|
||||
it.each(tiers)('%s never lets a free tier exceed paid', (_name, tier) => {
|
||||
for (const n of Object.values(tier.bySubscription))
|
||||
@@ -119,9 +105,9 @@ describe('the tiered subscription quotas', () => {
|
||||
});
|
||||
|
||||
it('holds an unrecognised plan to the paid base', () => {
|
||||
expect(
|
||||
limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan'),
|
||||
).toBe(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit);
|
||||
expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, 'some_plan')).toBe(
|
||||
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
|
||||
);
|
||||
expect(limitFor(EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, null)).toBe(
|
||||
EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER.limit,
|
||||
);
|
||||
@@ -145,9 +131,9 @@ describe('the delivery budgets', () => {
|
||||
expect(
|
||||
limitFor(EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT, plan),
|
||||
).toBe(matched);
|
||||
expect(limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan)).toBe(
|
||||
evaluated,
|
||||
);
|
||||
expect(
|
||||
limitFor(EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT, plan),
|
||||
).toBe(evaluated);
|
||||
expect(evaluated).toBeGreaterThanOrEqual(matched);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -53,12 +53,7 @@ export interface TieredLimit {
|
||||
bySubscription: Record<string, number>;
|
||||
}
|
||||
|
||||
/** Paid, free and temporary caps as a {@link TieredLimit}. Called by extensions. */
|
||||
export const tiered = (
|
||||
paid: number,
|
||||
free: number,
|
||||
temp: number,
|
||||
): TieredLimit => ({
|
||||
const tiered = (paid: number, free: number, temp: number): TieredLimit => ({
|
||||
limit: paid,
|
||||
bySubscription: {
|
||||
[DEFAULT_FREE_SUBSCRIPTION]: free,
|
||||
@@ -98,18 +93,6 @@ export interface TieredQuota {
|
||||
*/
|
||||
export const EVENTS_SESSION_SUBSCRIPTIONS_PER_SOCKET = 50;
|
||||
|
||||
/**
|
||||
* Live subscriptions one user may hold through a share handle someone else
|
||||
* minted for them, across all of their connections.
|
||||
*
|
||||
* Those rows sit in the owner's keyspace, where each one costs a filter
|
||||
* evaluation on the owner's writes. The count also takes in connections that
|
||||
* dropped and haven't been reaped yet, so the cap sits far above what
|
||||
* reconnects leave behind. The owner's own rows through the handle don't
|
||||
* count.
|
||||
*/
|
||||
export const EVENTS_SESSION_SUBSCRIPTIONS_PER_KV_HANDLE = 1000;
|
||||
|
||||
/**
|
||||
* Durable subscriptions one account may hold, across every app.
|
||||
*
|
||||
@@ -142,13 +125,6 @@ export const EVENTS_DURABLE_SUBSCRIPTIONS_MAX =
|
||||
*/
|
||||
export const SUSPENDED_ROW_TTL_DAYS = 30;
|
||||
|
||||
/**
|
||||
* Subscriptions stopped by a revoked grant that one account keeps for `list`.
|
||||
* They hold no quota slot, so past this the oldest are deleted when the account
|
||||
* subscribes again, keeping its rows bounded.
|
||||
*/
|
||||
export const EVENTS_REVOKED_SUBSCRIPTIONS_KEPT = 500;
|
||||
|
||||
/**
|
||||
* Subscribe calls per minute, per user.
|
||||
*
|
||||
@@ -179,13 +155,6 @@ export const EVENTS_LIST_LIMIT = userWindow('events:list', 120);
|
||||
*/
|
||||
export const EVENTS_ACK_LIMIT = userWindow('events:ack', 600);
|
||||
|
||||
/**
|
||||
* How long a connected client holds a queued delivery before the events worker
|
||||
* may take it. A client that has no handler for it, or whose handler fails,
|
||||
* declines it at once; this only bounds one that never answers.
|
||||
*/
|
||||
export const EVENTS_SOCKET_LEASE_MS = 15_000;
|
||||
|
||||
/**
|
||||
* Share-handle mint + revoke calls per minute, per user.
|
||||
*
|
||||
@@ -228,35 +197,17 @@ export const EVENTS_KV_HANDLES_PER_APP = tiered(10_000, 2_000, 0);
|
||||
export const EVENTS_KV_HANDLES_PER_HOLDER = 256;
|
||||
|
||||
/**
|
||||
* Recipients one account may share with, per minute and per hour. A call spends
|
||||
* one per recipient that doesn't already hold the share, on top of the per-call
|
||||
* mint budget.
|
||||
* Recipients one account may share with, per minute and per hour. A batch
|
||||
* spends one per recipient, on top of the per-call mint budget.
|
||||
*/
|
||||
export const KV_SHARE_GRANT_LIMITS = [
|
||||
userWindow('kvShare:grants:minute', 300),
|
||||
userWindow('kvShare:grants:hour', 2_000, 60 * 60_000),
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Recipients one account may name in share calls per minute, existing shares
|
||||
* included. Each one is looked up before the call knows whether it changes
|
||||
* anything, so a client that shares again on every load is bounded here.
|
||||
*/
|
||||
export const KV_SHARE_LOOKUP_LIMIT = userWindow('kvShare:lookups', 1_200);
|
||||
|
||||
/** Recipients one `POST /kv/shares` call may name. */
|
||||
export const KV_SHARE_BATCH_MAX = 50;
|
||||
|
||||
/**
|
||||
* Notifications one party of a share may send the other through share handles,
|
||||
* per minute and per day, in one app. Keeps one share from spending the
|
||||
* recipient's whole notification allowance.
|
||||
*/
|
||||
export const KV_SHARE_NOTIFY_LIMITS = [
|
||||
userWindow('kvShare:notify:minute', 30),
|
||||
userWindow('kvShare:notify:day', 500, 24 * 60 * 60_000),
|
||||
] as const;
|
||||
|
||||
// -- Handler surface -------------------------------------------------
|
||||
|
||||
/**
|
||||
@@ -353,11 +304,8 @@ export const EVENTS_KV_MATCHED_SUBSCRIPTIONS_PER_EVENT = tiered(512, 128, 128);
|
||||
*/
|
||||
export const EVENTS_KV_FILTER_EVALUATIONS_PER_EVENT = tiered(2048, 512, 512);
|
||||
|
||||
/**
|
||||
* Deliveries of one key-value change that carry its value inline, in the order
|
||||
* the fan-out serves them. Later ones carry the key alone.
|
||||
*/
|
||||
export const EVENTS_KV_VALUE_DELIVERIES_PER_EVENT = 128;
|
||||
/** Omit all inline KV values when the matched audience exceeds this count. */
|
||||
export const EVENTS_KV_VALUE_OMIT_MATCHED_SUBSCRIPTIONS = 128;
|
||||
|
||||
/**
|
||||
* Largest inline KV value in serialized bytes. Larger values are omitted;
|
||||
@@ -449,20 +397,16 @@ export const EVENTS_REGION_PENDING_CEILING = 1_000_000;
|
||||
// a handler that answers "no" is not. Retrying on a fixed cadence turns one
|
||||
// broken deploy into a permanent load on whatever is failing, so the wait
|
||||
// doubles per attempt up to a ceiling, and a run of failures stops the
|
||||
// subscription rather than retrying it forever. A run Puter could not start or
|
||||
// finish backs off the same way but is not the handler's failure, so it never
|
||||
// counts toward that run.
|
||||
// subscription rather than retrying it forever.
|
||||
|
||||
// Wait before the first retry of a delivery a handler could not take, the
|
||||
// longest wait, and the doubling between them. Shared with system jobs.
|
||||
export {
|
||||
deliveryBackoffMs,
|
||||
EVENTS_RETRY_BASE_MS,
|
||||
EVENTS_RETRY_MAX_MS,
|
||||
} from '../../util/retryBackoff.js';
|
||||
/** Wait before the first retry of a delivery a handler could not take. */
|
||||
export const EVENTS_RETRY_BASE_MS = 2_000;
|
||||
|
||||
/** Longest wait between retries, however many have failed. */
|
||||
export const EVENTS_RETRY_MAX_MS = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Handler failures in a row before a subscription is suspended. Counted per
|
||||
* Failures in a row before a subscription is suspended. Counted per
|
||||
* subscription and reset by the first delivery a handler takes, so an
|
||||
* occasional failure never accumulates into one.
|
||||
*/
|
||||
@@ -475,6 +419,13 @@ export const EVENTS_CONSECUTIVE_FAILURES = 5;
|
||||
*/
|
||||
export const EVENTS_FAILURE_COUNTER_TTL_MS = 60 * 60 * 1000;
|
||||
|
||||
/** How long to hold a delivery whose handler has failed `attempts` times. */
|
||||
export const deliveryBackoffMs = (attempts: number): number =>
|
||||
Math.min(
|
||||
EVENTS_RETRY_MAX_MS,
|
||||
EVENTS_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1),
|
||||
);
|
||||
|
||||
// -- Suspended backlog -----------------------------------------------
|
||||
//
|
||||
// A suspended subscription stops metering but keeps what it is owed, and that
|
||||
|
||||
@@ -1,274 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The schedule routes are thin delegates: without `services.schedules` every
|
||||
* one answers 503 `events_disabled`; with it, each hands its request over and
|
||||
* answers what it returns.
|
||||
*/
|
||||
|
||||
import type { Request, Response } from 'express';
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { makeActor, type Actor } from '../../core/actor.js';
|
||||
import { isHttpError } from '../../core/http/HttpError.js';
|
||||
import type {
|
||||
ScheduleServiceContract,
|
||||
ScheduleView,
|
||||
} from '../../core/schedules.js';
|
||||
import type { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { BroadcastController } from '../broadcast/BroadcastController.js';
|
||||
import type { EventsController } from './EventsController.js';
|
||||
|
||||
let server: PuterServer;
|
||||
let events: EventsController;
|
||||
let broadcast: BroadcastController;
|
||||
let actor: Actor;
|
||||
let saved: unknown;
|
||||
|
||||
const SCHEDULE_ID = '0b3a3c9e-9f61-4f0e-8d2a-2f1b6f8f2c11';
|
||||
|
||||
const view = {
|
||||
scheduleId: SCHEDULE_ID,
|
||||
kind: 'recurring',
|
||||
appUid: 'app-x',
|
||||
} as ScheduleView;
|
||||
|
||||
const slot = () => server.services as unknown as { schedules?: unknown };
|
||||
/** Put `schedules` in the slot, or leave the slot empty. */
|
||||
const setSchedules = (schedules: unknown): void => {
|
||||
if (schedules === undefined) delete slot().schedules;
|
||||
else slot().schedules = schedules;
|
||||
};
|
||||
|
||||
const fakeSchedules = () =>
|
||||
({
|
||||
onSubscriptionsChanged: vi.fn(async () => {}),
|
||||
createFor: vi.fn(async () => view),
|
||||
list: vi.fn(async () => ({
|
||||
items: [view],
|
||||
cursor: 'next',
|
||||
total: 3,
|
||||
})),
|
||||
get: vi.fn(async () => view),
|
||||
remove: vi.fn(async () => {}),
|
||||
receivePeerChanges: vi.fn(async () => ({ applied: 2 })),
|
||||
prepareSendAt: vi.fn(),
|
||||
createDeferredAppMessage: vi.fn(),
|
||||
}) satisfies ScheduleServiceContract;
|
||||
|
||||
const req = (init: {
|
||||
body?: unknown;
|
||||
query?: Record<string, unknown>;
|
||||
params?: Record<string, string>;
|
||||
}): Request =>
|
||||
({
|
||||
actor,
|
||||
body: init.body ?? {},
|
||||
query: init.query ?? {},
|
||||
params: init.params ?? {},
|
||||
headers: {},
|
||||
rawBody: Buffer.from(JSON.stringify(init.body ?? {})),
|
||||
}) as unknown as Request;
|
||||
|
||||
const res = () => {
|
||||
const sent: { status: number; body: unknown } = {
|
||||
status: 200,
|
||||
body: undefined,
|
||||
};
|
||||
const response = {
|
||||
json: vi.fn((body: unknown) => {
|
||||
sent.body = body;
|
||||
return response;
|
||||
}),
|
||||
status: vi.fn((code: number) => {
|
||||
sent.status = code;
|
||||
return response;
|
||||
}),
|
||||
setHeader: vi.fn(() => response),
|
||||
};
|
||||
return { response: response as unknown as Response, sent };
|
||||
};
|
||||
|
||||
/** Each route, called as the router would. */
|
||||
const routes: Array<[string, (response: Response) => Promise<void>]> = [
|
||||
[
|
||||
'POST /events/schedules',
|
||||
(r) => events.createSchedule(req({ body: { every: 60_000 } }), r),
|
||||
],
|
||||
[
|
||||
'GET /events/schedules',
|
||||
(r) => events.listSchedules(req({ query: { limit: '5' } }), r),
|
||||
],
|
||||
[
|
||||
'GET /events/schedules/:scheduleId',
|
||||
(r) =>
|
||||
events.getSchedule(req({ params: { scheduleId: SCHEDULE_ID } }), r),
|
||||
],
|
||||
[
|
||||
'POST /events/schedules/remove',
|
||||
(r) =>
|
||||
events.removeSchedule(
|
||||
req({ body: { scheduleId: SCHEDULE_ID } }),
|
||||
r,
|
||||
),
|
||||
],
|
||||
[
|
||||
'POST /broadcast/events/schedules',
|
||||
(r) =>
|
||||
broadcast.schedules(
|
||||
req({ body: { namespace: 'n', changes: [] } }),
|
||||
r,
|
||||
),
|
||||
],
|
||||
];
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
events = server.controllers.events as unknown as EventsController;
|
||||
broadcast = server.controllers.broadcast as unknown as BroadcastController;
|
||||
actor = makeActor({
|
||||
user: { id: 1, uuid: 'u-1', username: 'someone' },
|
||||
} as never);
|
||||
saved = slot().schedules;
|
||||
vi.spyOn(
|
||||
server.services.broadcast,
|
||||
'verifySignedRequest',
|
||||
).mockResolvedValue({ ok: true, peerId: 'peer' } as never);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setSchedules(saved);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
describe('without services.schedules', () => {
|
||||
it.each(routes)('%s answers 503 events_disabled', async (_name, call) => {
|
||||
setSchedules(undefined);
|
||||
const { response } = res();
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await call(response);
|
||||
} catch (err) {
|
||||
thrown = err;
|
||||
}
|
||||
expect(isHttpError(thrown)).toBe(true);
|
||||
expect(thrown).toMatchObject({
|
||||
statusCode: 503,
|
||||
legacyCode: 'events_disabled',
|
||||
// The designed answer, so it never pages.
|
||||
noAlarm: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses an unsigned peer batch before saying schedules are off', async () => {
|
||||
setSchedules(undefined);
|
||||
vi.spyOn(
|
||||
server.services.broadcast,
|
||||
'verifySignedRequest',
|
||||
).mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 401,
|
||||
message: 'bad signature',
|
||||
} as never);
|
||||
const { response, sent } = res();
|
||||
await broadcast.schedules(
|
||||
req({ body: { namespace: 'n', changes: [] } }),
|
||||
response,
|
||||
);
|
||||
expect(sent.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with services.schedules', () => {
|
||||
it('creates through createFor and answers its view', async () => {
|
||||
const schedules = fakeSchedules();
|
||||
setSchedules(schedules);
|
||||
const { response, sent } = res();
|
||||
await events.createSchedule(req({ body: { every: 60_000 } }), response);
|
||||
expect(schedules.createFor).toHaveBeenCalledWith(actor, {
|
||||
every: 60_000,
|
||||
});
|
||||
expect(sent.body).toEqual(view);
|
||||
});
|
||||
|
||||
it('lists with the limit capped, the cursor and includeTotal', async () => {
|
||||
const schedules = fakeSchedules();
|
||||
setSchedules(schedules);
|
||||
const { response, sent } = res();
|
||||
await events.listSchedules(
|
||||
req({
|
||||
query: { limit: '5000', cursor: 'c', includeTotal: 'true' },
|
||||
}),
|
||||
response,
|
||||
);
|
||||
expect(schedules.list).toHaveBeenCalledWith(actor, {
|
||||
limit: 200,
|
||||
cursor: 'c',
|
||||
includeTotal: true,
|
||||
});
|
||||
expect(sent.body).toEqual({ items: [view], cursor: 'next', total: 3 });
|
||||
});
|
||||
|
||||
it('reads one through get', async () => {
|
||||
const schedules = fakeSchedules();
|
||||
setSchedules(schedules);
|
||||
const { response, sent } = res();
|
||||
await events.getSchedule(
|
||||
req({ params: { scheduleId: SCHEDULE_ID } }),
|
||||
response,
|
||||
);
|
||||
expect(schedules.get).toHaveBeenCalledWith(actor, SCHEDULE_ID);
|
||||
expect(sent.body).toEqual(view);
|
||||
});
|
||||
|
||||
it('removes through remove and answers {}', async () => {
|
||||
const schedules = fakeSchedules();
|
||||
setSchedules(schedules);
|
||||
const { response, sent } = res();
|
||||
await events.removeSchedule(
|
||||
req({ body: { scheduleId: SCHEDULE_ID } }),
|
||||
response,
|
||||
);
|
||||
expect(schedules.remove).toHaveBeenCalledWith(actor, SCHEDULE_ID);
|
||||
expect(sent.body).toEqual({});
|
||||
});
|
||||
|
||||
it('hands a signed peer batch to receivePeerChanges', async () => {
|
||||
const schedules = fakeSchedules();
|
||||
setSchedules(schedules);
|
||||
const { response, sent } = res();
|
||||
const body = { namespace: 'n', changes: [] };
|
||||
await broadcast.schedules(req({ body }), response);
|
||||
expect(schedules.receivePeerChanges).toHaveBeenCalledWith(body);
|
||||
expect(sent).toEqual({ status: 200, body: { ok: true, applied: 2 } });
|
||||
});
|
||||
});
|
||||
@@ -25,13 +25,8 @@
|
||||
|
||||
import MockRedis from 'ioredis-mock';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
|
||||
import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
|
||||
import {
|
||||
EventsWorkerDeployer,
|
||||
LocalEventsInvokeTransport,
|
||||
type EventsDeployOutcome,
|
||||
} from './workerDeploy.js';
|
||||
import { EventsWorkerDeployer } from './workerDeploy.js';
|
||||
import { eventsWorkerScript } from '../../services/events/workerRuntime.js';
|
||||
import { handlerSetHash } from '../../services/events/workerSource.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
@@ -54,8 +49,6 @@ type Layers = ConstructorParameters<typeof EventsWorkerDeployer>[0];
|
||||
const makeLayers = (owners: Record<string, Owner | null>) => {
|
||||
const redis = new MockRedis.Cluster(['redis://localhost:7001']);
|
||||
const created: string[] = [];
|
||||
/** Apps whose handler sources were read, once per read. */
|
||||
const sourceReads: string[] = [];
|
||||
const layers = {
|
||||
config: {
|
||||
events: { workerRuntime: true, internalSecret: 'sekret' },
|
||||
@@ -67,20 +60,16 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
|
||||
clients: { redis },
|
||||
stores: {
|
||||
eventHandler: {
|
||||
setForApp: async () => [{ name: 'h', sourceHash: SOURCE_HASH }],
|
||||
allForApp: async (appUid: string) => {
|
||||
sourceReads.push(appUid);
|
||||
return [
|
||||
{
|
||||
appUid,
|
||||
name: 'h',
|
||||
source: 'async () => {}',
|
||||
sourceHash: SOURCE_HASH,
|
||||
createdAt: 0,
|
||||
updatedAt: 0,
|
||||
},
|
||||
];
|
||||
},
|
||||
allForApp: async (appUid: string) => [
|
||||
{
|
||||
appUid,
|
||||
name: 'h',
|
||||
source: 'async () => {}',
|
||||
sourceHash: SOURCE_HASH,
|
||||
createdAt: 0,
|
||||
updatedAt: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
app: {
|
||||
getByUid: async (appUid: string) => {
|
||||
@@ -115,12 +104,7 @@ const makeLayers = (owners: Record<string, Owner | null>) => {
|
||||
},
|
||||
},
|
||||
};
|
||||
return {
|
||||
layers: layers as unknown as Layers,
|
||||
created,
|
||||
redis,
|
||||
sourceReads,
|
||||
};
|
||||
return { layers: layers as unknown as Layers, created, redis };
|
||||
};
|
||||
|
||||
describe('EventsWorkerDeployer.ensure', () => {
|
||||
@@ -143,10 +127,8 @@ describe('EventsWorkerDeployer.ensure', () => {
|
||||
expect(b).toBe('no-owner');
|
||||
});
|
||||
|
||||
it('refuses once the app has deployed past the hourly cap, before reading or building any source', async () => {
|
||||
const { layers, redis, sourceReads, created } = makeLayers({
|
||||
'app-c': { ownerUserId: 3 },
|
||||
});
|
||||
it('refuses once the app has deployed past the hourly cap', async () => {
|
||||
const { layers, redis } = makeLayers({ 'app-c': { ownerUserId: 3 } });
|
||||
const deployer = new EventsWorkerDeployer(layers);
|
||||
const hour = new Date().toISOString().slice(0, 13);
|
||||
await redis.set(
|
||||
@@ -155,24 +137,6 @@ describe('EventsWorkerDeployer.ensure', () => {
|
||||
);
|
||||
|
||||
expect(await deployer.ensure('app-c', SCRIPT)).toBe('throttled');
|
||||
expect(sourceReads).toEqual([]);
|
||||
expect(created).toEqual([]);
|
||||
});
|
||||
|
||||
it('answers a stale script without spending the hourly cap', async () => {
|
||||
const { layers, redis, sourceReads } = makeLayers({
|
||||
'app-e': { ownerUserId: 5 },
|
||||
});
|
||||
const deployer = new EventsWorkerDeployer(layers);
|
||||
const stale = eventsWorkerScript(
|
||||
handlerSetHash([{ name: 'h', sourceHash: 'b'.repeat(64) }]),
|
||||
'',
|
||||
);
|
||||
|
||||
expect(await deployer.ensure('app-e', stale)).toBe('stale');
|
||||
expect(sourceReads).toEqual([]);
|
||||
const hour = new Date().toISOString().slice(0, 13);
|
||||
expect(await redis.get(`ev:deploys:{app-e}:${hour}`)).toBeNull();
|
||||
});
|
||||
|
||||
it('deploys normally under the cap', async () => {
|
||||
@@ -181,151 +145,3 @@ describe('EventsWorkerDeployer.ensure', () => {
|
||||
expect(await deployer.ensure('app-d', SCRIPT)).toBe('deployed');
|
||||
});
|
||||
});
|
||||
|
||||
/** A server's schedules with a deploy gate that records and can refuse. */
|
||||
const withGate = (
|
||||
layers: Layers,
|
||||
refuse: (capped: boolean) => boolean = () => false,
|
||||
) => {
|
||||
const gated: Array<{ script: string; capped: boolean }> = [];
|
||||
const gate = {
|
||||
gateDeploy: async (
|
||||
script: string,
|
||||
options: { capped: boolean },
|
||||
deploy: () => Promise<EventsDeployOutcome>,
|
||||
): Promise<EventsDeployOutcome> => {
|
||||
gated.push({ script, capped: options.capped });
|
||||
return refuse(options.capped) ? 'busy' : deploy();
|
||||
},
|
||||
};
|
||||
const deployer = new EventsWorkerDeployer({
|
||||
...layers,
|
||||
services: { ...layers.services, schedules: gate },
|
||||
} as unknown as Layers);
|
||||
return { deployer, gated };
|
||||
};
|
||||
|
||||
describe('the schedules` deploy gate', () => {
|
||||
it('is skipped without schedules, or with schedules that have no gate', async () => {
|
||||
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
|
||||
await expect(
|
||||
new EventsWorkerDeployer(layers).ensure('app-x', SCRIPT, {
|
||||
lane: 'schedule',
|
||||
}),
|
||||
).resolves.toBe('deployed');
|
||||
const gateless = new EventsWorkerDeployer({
|
||||
...layers,
|
||||
services: { ...layers.services, schedules: {} },
|
||||
} as unknown as Layers);
|
||||
await expect(
|
||||
gateless.ensure('app-x', SCRIPT, { lane: 'schedule' }),
|
||||
).resolves.toBe('deployed');
|
||||
expect(created).toEqual([SCRIPT, SCRIPT]);
|
||||
});
|
||||
|
||||
it('takes every deploy, capped only for a schedule run', async () => {
|
||||
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
|
||||
const { deployer, gated } = withGate(layers);
|
||||
|
||||
await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
|
||||
'deployed',
|
||||
);
|
||||
await expect(
|
||||
deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
|
||||
).resolves.toBe('deployed');
|
||||
expect(gated).toEqual([
|
||||
{ script: SCRIPT, capped: false },
|
||||
{ script: SCRIPT, capped: true },
|
||||
]);
|
||||
expect(created).toEqual([SCRIPT, SCRIPT]);
|
||||
});
|
||||
|
||||
it('answers a refused schedule run busy, and a delivery still deploys', async () => {
|
||||
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
|
||||
const { deployer } = withGate(layers, (capped) => capped);
|
||||
|
||||
await expect(
|
||||
deployer.ensure('app-x', SCRIPT, { lane: 'schedule' }),
|
||||
).resolves.toBe('busy');
|
||||
await expect(deployer.ensure('app-x', SCRIPT)).resolves.toBe(
|
||||
'deployed',
|
||||
);
|
||||
expect(created).toEqual([SCRIPT]);
|
||||
});
|
||||
|
||||
it('lets deliveries that join a schedule run`s refused deploy deploy once', async () => {
|
||||
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
|
||||
const { deployer, gated } = withGate(layers, (capped) => capped);
|
||||
|
||||
const scheduled = deployer.ensure('app-x', SCRIPT, {
|
||||
lane: 'schedule',
|
||||
});
|
||||
const deliveries = [
|
||||
deployer.ensure('app-x', SCRIPT),
|
||||
deployer.ensure('app-x', SCRIPT),
|
||||
];
|
||||
await expect(scheduled).resolves.toBe('busy');
|
||||
await expect(Promise.all(deliveries)).resolves.toEqual([
|
||||
'deployed',
|
||||
'deployed',
|
||||
]);
|
||||
expect(created).toEqual([SCRIPT]);
|
||||
expect(gated.map((g) => g.capped)).toEqual([true, false]);
|
||||
});
|
||||
|
||||
it('lets a schedule run join a delivery`s deploy in flight', async () => {
|
||||
const { layers, created } = makeLayers({ 'app-x': { ownerUserId: 1 } });
|
||||
const { deployer, gated } = withGate(layers, (capped) => capped);
|
||||
|
||||
const delivery = deployer.ensure('app-x', SCRIPT);
|
||||
const scheduled = deployer.ensure('app-x', SCRIPT, {
|
||||
lane: 'schedule',
|
||||
});
|
||||
await expect(Promise.all([delivery, scheduled])).resolves.toEqual([
|
||||
'deployed',
|
||||
'deployed',
|
||||
]);
|
||||
expect(gated).toEqual([{ script: SCRIPT, capped: false }]);
|
||||
expect(created).toEqual([SCRIPT]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LocalEventsInvokeTransport', () => {
|
||||
it('renders each request as it goes out, after a cold deploy', async () => {
|
||||
const bodies: string[] = [];
|
||||
let resident = false;
|
||||
const transport = new LocalEventsInvokeTransport(
|
||||
{
|
||||
localworkerservice: {
|
||||
dispatchEventsWorker: async (request: Request) => {
|
||||
bodies.push(await request.text());
|
||||
if (!resident) return null;
|
||||
return new Response(null, {
|
||||
status: 200,
|
||||
headers: { 'x-puter-events-handled': '1' },
|
||||
});
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
{
|
||||
ensure: async () => {
|
||||
resident = true;
|
||||
return 'deployed';
|
||||
},
|
||||
} as unknown as EventsWorkerDeployer,
|
||||
);
|
||||
let renders = 0;
|
||||
const call: EventsInvokeCall = {
|
||||
script: SCRIPT,
|
||||
appUid: 'app-x',
|
||||
key: 'k',
|
||||
body: '{"n":0}',
|
||||
timeoutMs: 5000,
|
||||
render: () => `{"n":${++renders}}`,
|
||||
};
|
||||
await expect(transport.send(call)).resolves.toMatchObject({
|
||||
status: 200,
|
||||
});
|
||||
expect(bodies).toEqual(['{"n":1}', '{"n":2}']);
|
||||
});
|
||||
});
|
||||
@@ -26,27 +26,18 @@ import {
|
||||
INTERNAL_ADMISSION_BYPASS,
|
||||
INTERNAL_DEPLOY_TARGET,
|
||||
} from '../../drivers/workers/WorkerDriver.js';
|
||||
import type {
|
||||
EventsInvokeAnswer,
|
||||
EventsInvokeCall,
|
||||
InvokeLane,
|
||||
} from '../../clients/events/EventsWorkerInvokerClient.js';
|
||||
import type { EventsInvokeCall } from '../../clients/events/EventsWorkerInvokerClient.js';
|
||||
import {
|
||||
EVENTS_HANDLED_HEADER,
|
||||
EVENTS_INVOKE_PATH,
|
||||
answerFrom,
|
||||
failedAnswer,
|
||||
} from '../../clients/events/EventsWorkerInvokerClient.js';
|
||||
import type { puterServices } from '../../services/index.js';
|
||||
import type { IExtensionServiceInstances } from '../../services/types.js';
|
||||
import {
|
||||
eventsInvokeKey,
|
||||
eventsWorkerScope,
|
||||
eventsWorkerScript,
|
||||
} from '../../services/events/workerRuntime.js';
|
||||
import {
|
||||
generateEventsWorkerSource,
|
||||
handlerSetHash,
|
||||
} from '../../services/events/workerSource.js';
|
||||
import { generateEventsWorkerSource } from '../../services/events/workerSource.js';
|
||||
import type { puterStores } from '../../stores/index.js';
|
||||
import type { IConfig, LayerInstances } from '../../types.js';
|
||||
import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
|
||||
@@ -63,25 +54,13 @@ import { EVENTS_WORKER_DEPLOYS_PER_HOUR } from './limits.js';
|
||||
|
||||
/** Why a deploy could not happen, for the callers that answer differently. */
|
||||
export type EventsDeployOutcome =
|
||||
| 'deployed'
|
||||
| 'stale'
|
||||
| 'no-handlers'
|
||||
| 'no-owner'
|
||||
| 'throttled'
|
||||
| 'failed'
|
||||
| 'busy';
|
||||
|
||||
/** Whose call asked for the deploy: a schedule run's is held to a share. */
|
||||
export interface EventsDeployOptions {
|
||||
lane?: InvokeLane;
|
||||
}
|
||||
'deployed' | 'stale' | 'no-handlers' | 'no-owner' | 'throttled' | 'failed';
|
||||
|
||||
interface DeployLayers {
|
||||
config: IConfig;
|
||||
clients: LayerInstances<typeof puterClients>;
|
||||
stores: LayerInstances<typeof puterStores>;
|
||||
services: LayerInstances<typeof puterServices> &
|
||||
Pick<IExtensionServiceInstances, 'schedules'>;
|
||||
services: LayerInstances<typeof puterServices>;
|
||||
drivers: LayerInstances<typeof puterDrivers>;
|
||||
}
|
||||
|
||||
@@ -91,12 +70,6 @@ interface DeployLayers {
|
||||
*/
|
||||
const DEPLOY_THROTTLE_TTL_SECONDS = 2 * 60 * 60;
|
||||
|
||||
interface InFlightDeploy {
|
||||
/** A schedule run's, which the region's deploy slots may refuse. */
|
||||
capped: boolean;
|
||||
promise: Promise<EventsDeployOutcome>;
|
||||
}
|
||||
|
||||
interface DeployResult {
|
||||
success?: boolean;
|
||||
errors?: unknown[];
|
||||
@@ -105,7 +78,7 @@ interface DeployResult {
|
||||
export class EventsWorkerDeployer {
|
||||
readonly #layers: DeployLayers;
|
||||
/** Collapses the burst of misses one cold app produces into one deploy. */
|
||||
readonly #inFlight = new Map<string, InFlightDeploy>();
|
||||
readonly #inFlight = new Map<string, Promise<EventsDeployOutcome>>();
|
||||
|
||||
constructor(layers: DeployLayers) {
|
||||
this.#layers = layers;
|
||||
@@ -124,79 +97,15 @@ export class EventsWorkerDeployer {
|
||||
* A request for anything else is `stale` — the set moved on, and the next
|
||||
* delivery resolves the new script rather than this one being revived.
|
||||
*/
|
||||
ensure(
|
||||
appUid: string,
|
||||
script: string,
|
||||
options: EventsDeployOptions = {},
|
||||
): Promise<EventsDeployOutcome> {
|
||||
ensure(appUid: string, script: string): Promise<EventsDeployOutcome> {
|
||||
const key = `${appUid}|${script}`;
|
||||
const capped = options.lane === 'schedule';
|
||||
const existing = this.#inFlight.get(key);
|
||||
if (!existing) return this.#start(key, appUid, script, options);
|
||||
if (capped || !existing.capped) return existing.promise;
|
||||
return this.#afterCapped(
|
||||
existing.promise,
|
||||
key,
|
||||
appUid,
|
||||
script,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
#start(
|
||||
key: string,
|
||||
appUid: string,
|
||||
script: string,
|
||||
options: EventsDeployOptions,
|
||||
): Promise<EventsDeployOutcome> {
|
||||
const entry: InFlightDeploy = {
|
||||
capped: options.lane === 'schedule',
|
||||
promise: this.#gated(appUid, script, options).finally(() => {
|
||||
if (this.#inFlight.get(key) === entry)
|
||||
this.#inFlight.delete(key);
|
||||
}),
|
||||
};
|
||||
this.#inFlight.set(key, entry);
|
||||
return entry.promise;
|
||||
}
|
||||
|
||||
/**
|
||||
* A delivery that joined a schedule run's deploy: `busy` means no deploy
|
||||
* started, and a delivery is never refused a slot, so it asks again.
|
||||
*/
|
||||
async #afterCapped(
|
||||
capped: Promise<EventsDeployOutcome>,
|
||||
key: string,
|
||||
appUid: string,
|
||||
script: string,
|
||||
options: EventsDeployOptions,
|
||||
): Promise<EventsDeployOutcome> {
|
||||
const outcome = await capped;
|
||||
if (outcome !== 'busy') return outcome;
|
||||
const current = this.#inFlight.get(key);
|
||||
if (current && !current.capped) return current.promise;
|
||||
return this.#start(key, appUid, script, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every deploy goes through the schedules' deploy gate when a server has
|
||||
* one: a per-script lock, so a burst of runs of an evicted script deploys
|
||||
* it once, and a region-wide cap that a schedule run's deploy (`capped`)
|
||||
* may be refused with `busy`. A delivery's deploy is never refused.
|
||||
*/
|
||||
#gated(
|
||||
appUid: string,
|
||||
script: string,
|
||||
options: EventsDeployOptions,
|
||||
): Promise<EventsDeployOutcome> {
|
||||
const deploy = () => this.#deploy(appUid, script);
|
||||
return (
|
||||
this.#layers.services.schedules?.gateDeploy?.(
|
||||
script,
|
||||
{ capped: options.lane === 'schedule' },
|
||||
deploy,
|
||||
) ?? deploy()
|
||||
if (existing) return existing;
|
||||
const started = this.#deploy(appUid, script).finally(() =>
|
||||
this.#inFlight.delete(key),
|
||||
);
|
||||
this.#inFlight.set(key, started);
|
||||
return started;
|
||||
}
|
||||
|
||||
async #deploy(
|
||||
@@ -204,24 +113,20 @@ export class EventsWorkerDeployer {
|
||||
script: string,
|
||||
): Promise<EventsDeployOutcome> {
|
||||
const { stores, drivers } = this.#layers;
|
||||
const scope = eventsWorkerScope(this.#layers.config);
|
||||
|
||||
// Names and hashes are enough to tell a stale request, so the set's
|
||||
// sources are read and built only once the throttle has let it through.
|
||||
const set = await stores.eventHandler.setForApp(appUid);
|
||||
if (set.length === 0) return 'no-handlers';
|
||||
const handlers = await stores.eventHandler.allForApp(appUid);
|
||||
if (handlers.length === 0) return 'no-handlers';
|
||||
|
||||
const generated = generateEventsWorkerSource(handlers);
|
||||
// Two apps that published byte-identical sets share one script, which
|
||||
// is sound: the source is the same, and everything per-app arrives with
|
||||
// the invocation. Whoever deploys first owns it.
|
||||
if (eventsWorkerScript(handlerSetHash(set), scope) !== script)
|
||||
return 'stale';
|
||||
|
||||
if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
|
||||
|
||||
const handlers = await stores.eventHandler.allForApp(appUid);
|
||||
const generated = generateEventsWorkerSource(handlers);
|
||||
// The set can move between the two reads.
|
||||
if (eventsWorkerScript(generated.setHash, scope) !== script)
|
||||
if (
|
||||
eventsWorkerScript(
|
||||
generated.setHash,
|
||||
eventsWorkerScope(this.#layers.config),
|
||||
) !== script
|
||||
)
|
||||
return 'stale';
|
||||
if (generated.tooLarge) {
|
||||
console.error(
|
||||
@@ -235,6 +140,8 @@ export class EventsWorkerDeployer {
|
||||
generated.broken.join(', '),
|
||||
);
|
||||
|
||||
if (await this.#tooManyDeploysThisHour(appUid)) return 'throttled';
|
||||
|
||||
const owner = await this.#owner(appUid);
|
||||
if (!owner) return 'no-owner';
|
||||
|
||||
@@ -382,9 +289,11 @@ export class LocalEventsInvokeTransport {
|
||||
this.#deployer = deployer;
|
||||
}
|
||||
|
||||
async send(call: EventsInvokeCall): Promise<EventsInvokeAnswer> {
|
||||
// Built per attempt, so a send time stamped into the body is taken
|
||||
// after a cold deploy rather than before it.
|
||||
async send(call: EventsInvokeCall): Promise<{
|
||||
status: number | null;
|
||||
handled?: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
const request = (): Request =>
|
||||
new Request(
|
||||
`http://${call.script}.workers.puter.localhost${EVENTS_INVOKE_PATH}`,
|
||||
@@ -394,7 +303,7 @@ export class LocalEventsInvokeTransport {
|
||||
'content-type': 'application/json',
|
||||
'x-puter-events-key': call.key,
|
||||
},
|
||||
body: call.render ? call.render() : call.body,
|
||||
body: call.body,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -405,21 +314,19 @@ export class LocalEventsInvokeTransport {
|
||||
const outcome = await this.#deployer.ensure(
|
||||
call.appUid,
|
||||
call.script,
|
||||
{ lane: call.lane },
|
||||
);
|
||||
if (outcome !== 'deployed')
|
||||
return {
|
||||
status: null,
|
||||
error: `deploy: ${outcome}`,
|
||||
...(outcome === 'busy' ? { deployBusy: true } : {}),
|
||||
};
|
||||
return { status: null, error: `deploy: ${outcome}` };
|
||||
|
||||
const retried = await this.#dispatch(request(), call.timeoutMs);
|
||||
if (retried === null)
|
||||
return { status: null, error: 'script not resident' };
|
||||
return retried;
|
||||
} catch (err) {
|
||||
return failedAnswer(err);
|
||||
return {
|
||||
status: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +337,7 @@ export class LocalEventsInvokeTransport {
|
||||
async #dispatch(
|
||||
request: Request,
|
||||
timeoutMs: number,
|
||||
): Promise<EventsInvokeAnswer | null> {
|
||||
): Promise<{ status: number; handled: boolean } | null> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeout = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(() => resolve('timeout'), timeoutMs);
|
||||
@@ -441,10 +348,12 @@ export class LocalEventsInvokeTransport {
|
||||
this.#services.localworkerservice.dispatchEventsWorker(request),
|
||||
timeout,
|
||||
]);
|
||||
if (response === 'timeout')
|
||||
throw new DOMException('invocation timed out', 'TimeoutError');
|
||||
if (response === 'timeout') throw new Error('invocation timed out');
|
||||
if (response === null) return null;
|
||||
return answerFrom(response);
|
||||
return {
|
||||
status: response.status,
|
||||
handled: response.headers.get(EVENTS_HANDLED_HEADER) === '1',
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
describe('recursive readdir with offset', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = async (path: string, token: string, body: unknown) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
it('skips entries before applying the recursive page limit', async () => {
|
||||
const { username, token } = env.users.user;
|
||||
const base = `/${username}/Documents/rec-offset`;
|
||||
for (const path of [base, `${base}/a`, `${base}/b`, `${base}/c`]) {
|
||||
expect((await post('/fs/mkdir', token, { path })).status).toBe(200);
|
||||
}
|
||||
|
||||
const list = async (body: Record<string, unknown>) => {
|
||||
const response = await post('/fs/readdir', token, {
|
||||
path: base,
|
||||
recursive: true,
|
||||
...body,
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
const page = (await response.json()) as {
|
||||
items: Array<{ name: string }>;
|
||||
};
|
||||
return page.items.map((item) => item.name);
|
||||
};
|
||||
|
||||
expect(await list({ limit: 3 })).toEqual(['a', 'b', 'c']);
|
||||
// Non-recursive honours offset.
|
||||
const flat = await post('/fs/readdir', token, {
|
||||
path: base,
|
||||
limit: 1,
|
||||
offset: 2,
|
||||
});
|
||||
expect(
|
||||
((await flat.json()) as Array<{ name: string }>).map((e) => e.name),
|
||||
).toEqual(['c']);
|
||||
expect(await list({ limit: 1, offset: 2 })).toEqual(['c']);
|
||||
expect(await list({ offset: 2 })).toEqual(['c']);
|
||||
expect(await list({ offset: 20 })).toEqual([]);
|
||||
expect(await list({ limit: 1, offset: 1, sortOrder: 'desc' })).toEqual([
|
||||
'b',
|
||||
]);
|
||||
const first = await post('/fs/readdir', token, {
|
||||
path: base,
|
||||
recursive: true,
|
||||
limit: 1,
|
||||
offset: 1,
|
||||
});
|
||||
const page = await first.json();
|
||||
expect(page.items.map((item: { name: string }) => item.name)).toEqual([
|
||||
'b',
|
||||
]);
|
||||
expect(page.cursor).toBeTypeOf('string');
|
||||
expect(await list({ limit: 1, cursor: page.cursor })).toEqual(['c']);
|
||||
// A cursor wins over offset, and an invalid offset is ignored, as
|
||||
// recursive listings did before they honoured offset.
|
||||
expect(
|
||||
await list({ limit: 1, cursor: page.cursor, offset: 1 }),
|
||||
).toEqual(['c']);
|
||||
expect(await list({ limit: 1, offset: -1 })).toEqual(['a']);
|
||||
});
|
||||
});
|
||||
@@ -1335,7 +1335,6 @@ export class FSController extends PuterController {
|
||||
parent.path,
|
||||
{
|
||||
limit,
|
||||
offset,
|
||||
cursor:
|
||||
typeof body.cursor === 'string'
|
||||
? body.cursor
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { makeActor } from '../../core/actor.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
describe('/auth/check-app-acl', () => {
|
||||
let env: PuterTestEnv;
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = (path: string, body: unknown, token: string) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const makeDirAndApp = async () => {
|
||||
const { username, token } = env.users.user;
|
||||
const server = env.server;
|
||||
const user = (await server.stores.user.getByUsername(username))!;
|
||||
const path = `/${username}/Documents/cacl-${uuidv4()}`;
|
||||
expect((await post('/fs/mkdir', { path }, token)).status).toBe(200);
|
||||
const entry = (await server.stores.fsEntry.getEntryByPath(path))!;
|
||||
|
||||
const app = await (
|
||||
server.stores.app.create as unknown as (
|
||||
f: Record<string, unknown>,
|
||||
o: { ownerUserId: number },
|
||||
) => Promise<{ uid: string; id: number }>
|
||||
)(
|
||||
{
|
||||
name: `cacl-${uuidv4()}`,
|
||||
title: 'Check app acl',
|
||||
index_url: `https://cacl-${uuidv4()}.test/`,
|
||||
},
|
||||
{ ownerUserId: user.id },
|
||||
);
|
||||
return { user, path, entry, app };
|
||||
};
|
||||
|
||||
it('allows a granted file without poisoning the app permission cache', async () => {
|
||||
const { token } = env.users.user;
|
||||
const server = env.server;
|
||||
const { user, path, entry, app } = await makeDirAndApp();
|
||||
|
||||
const grant = await post(
|
||||
'/auth/grant-user-app',
|
||||
{ app_uid: app.uid, permission: `fs:${entry.uuid}:read` },
|
||||
token,
|
||||
);
|
||||
expect(grant.status).toBe(200);
|
||||
|
||||
const res = await post(
|
||||
'/auth/check-app-acl',
|
||||
{ subject: { path }, app: app.uid, mode: 'read' },
|
||||
token,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(await res.json()).toEqual({ allowed: true });
|
||||
|
||||
const asApp = makeActor({ user, app: { uid: app.uid, id: app.id } });
|
||||
const descriptor = () => server.services.acl.fsDescriptor(path);
|
||||
expect(
|
||||
await server.services.acl.check(asApp, descriptor(), 'read'),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('denies an app with no grant', async () => {
|
||||
const { path, app } = await makeDirAndApp();
|
||||
const res = await post(
|
||||
'/auth/check-app-acl',
|
||||
{ subject: { path }, app: app.uid, mode: 'read' },
|
||||
env.users.user.token,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(await res.json()).toEqual({ allowed: false });
|
||||
});
|
||||
|
||||
it('refuses app and access tokens', async () => {
|
||||
const { token, apiToken } = env.users.user;
|
||||
const { path, entry, app } = await makeDirAndApp();
|
||||
const grant = await post(
|
||||
'/auth/grant-user-app',
|
||||
{ app_uid: app.uid, permission: `fs:${entry.uuid}:read` },
|
||||
token,
|
||||
);
|
||||
expect(grant.status).toBe(200);
|
||||
|
||||
const minted = await post(
|
||||
'/auth/get-user-app-token',
|
||||
{ app_uid: app.uid },
|
||||
token,
|
||||
);
|
||||
expect(minted.status).toBe(200);
|
||||
const appToken = ((await minted.json()) as { token: string }).token;
|
||||
|
||||
for (const credential of [appToken, apiToken]) {
|
||||
const res = await post(
|
||||
'/auth/check-app-acl',
|
||||
{ subject: { path }, app: app.uid, mode: 'read' },
|
||||
credential,
|
||||
);
|
||||
expect(res.status).toBe(403);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,204 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = (path: string, body: unknown) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${env.users.user.token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const write = (path: string, content: string) =>
|
||||
post('/fs/write', {
|
||||
fileMetadata: { path, size: content.length, contentType: 'text/plain' },
|
||||
fileContent: content,
|
||||
});
|
||||
|
||||
const read = async (path: string) => {
|
||||
const url = new URL('/read', env.apiOrigin);
|
||||
url.searchParams.set('file', path);
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${env.users.user.token}` },
|
||||
});
|
||||
return response.text();
|
||||
};
|
||||
|
||||
const makeDir = async (name: string) => {
|
||||
const path = `/${env.users.user.username}/Documents/${name}-${randomUUID().slice(0, 8)}`;
|
||||
expect((await post('/fs/mkdir', { path })).status).toBe(200);
|
||||
return path;
|
||||
};
|
||||
|
||||
describe('legacy /copy onto an existing file', () => {
|
||||
it('replaces the file with overwrite', async () => {
|
||||
const dir = await makeDir('copy-onto');
|
||||
expect((await write(`${dir}/a.txt`, 'aaa')).status).toBe(200);
|
||||
expect((await write(`${dir}/b.txt`, 'bbbbbb')).status).toBe(200);
|
||||
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination: `${dir}/b.txt`,
|
||||
overwrite: true,
|
||||
});
|
||||
expect(copy.status).toBe(200);
|
||||
const [result] = (await copy.json()) as Array<{
|
||||
copied: { name: string; path: string };
|
||||
overwritten?: { name: string };
|
||||
}>;
|
||||
expect(result!.copied.name).toBe('b.txt');
|
||||
expect(result!.overwritten?.name).toBe('b.txt');
|
||||
expect(await read(`${dir}/b.txt`)).toBe('aaa');
|
||||
expect(await read(`${dir}/a.txt`)).toBe('aaa');
|
||||
});
|
||||
|
||||
it('refuses without overwrite and leaves the file alone', async () => {
|
||||
const dir = await makeDir('copy-onto-refused');
|
||||
await write(`${dir}/a.txt`, 'aaa');
|
||||
await write(`${dir}/b.txt`, 'bbbbbb');
|
||||
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination: `${dir}/b.txt`,
|
||||
});
|
||||
expect(copy.status).toBe(409);
|
||||
expect(((await copy.json()) as { code: string }).code).toBe(
|
||||
'item_with_same_name_exists',
|
||||
);
|
||||
expect(await read(`${dir}/b.txt`)).toBe('bbbbbb');
|
||||
});
|
||||
|
||||
it('still copies into a directory destination under the same name', async () => {
|
||||
const dir = await makeDir('copy-into');
|
||||
await write(`${dir}/a.txt`, 'aaa');
|
||||
expect((await post('/fs/mkdir', { path: `${dir}/sub` })).status).toBe(
|
||||
200,
|
||||
);
|
||||
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination: `${dir}/sub`,
|
||||
});
|
||||
expect(copy.status).toBe(200);
|
||||
expect(await read(`${dir}/sub/a.txt`)).toBe('aaa');
|
||||
});
|
||||
|
||||
it('refuses to overwrite a file with itself', async () => {
|
||||
const dir = await makeDir('copy-self');
|
||||
await write(`${dir}/a.txt`, 'aaa');
|
||||
|
||||
for (const destination of [`${dir}/a.txt`, dir]) {
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination,
|
||||
overwrite: true,
|
||||
});
|
||||
expect(copy.status).toBe(400);
|
||||
}
|
||||
expect(await read(`${dir}/a.txt`)).toBe('aaa');
|
||||
});
|
||||
});
|
||||
|
||||
describe('signed folder listing', () => {
|
||||
const signedListing = async (path: string) => {
|
||||
const signed = await post('/sign', {
|
||||
items: [{ path, action: 'read' }],
|
||||
});
|
||||
expect(signed.status).toBe(200);
|
||||
const readUrl = new URL(
|
||||
(await signed.json()).signatures[0].read_url as string,
|
||||
);
|
||||
return fetch(
|
||||
new URL(`${readUrl.pathname}${readUrl.search}`, env.apiOrigin),
|
||||
);
|
||||
};
|
||||
|
||||
it('lists every child of a folder larger than one query', async () => {
|
||||
const dir = await makeDir('big');
|
||||
const parent = (await env.server.stores.fsEntry.getEntryByPath(dir))!;
|
||||
const total = 10_050;
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
for (let start = 0; start < total; start += 500) {
|
||||
const rows: unknown[][] = [];
|
||||
for (let i = start; i < Math.min(total, start + 500); i++) {
|
||||
const name = `f${String(i).padStart(5, '0')}.txt`;
|
||||
rows.push([
|
||||
randomUUID(),
|
||||
parent.userId,
|
||||
parent.id,
|
||||
parent.uuid,
|
||||
name,
|
||||
`${dir}/${name}`,
|
||||
0,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
0,
|
||||
]);
|
||||
}
|
||||
await env.server.clients.db.write(
|
||||
`INSERT INTO fsentries (uuid, user_id, parent_id, parent_uid, name, path, is_dir, modified, created, accessed, size) VALUES ${rows.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
|
||||
rows.flat(),
|
||||
);
|
||||
}
|
||||
|
||||
const response = await signedListing(dir);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toMatch(
|
||||
/^application\/json/,
|
||||
);
|
||||
const children = (await response.json()) as Array<{
|
||||
fsentry_name: string;
|
||||
write_url?: string;
|
||||
}>;
|
||||
expect(children).toHaveLength(total);
|
||||
expect(children[0]!.fsentry_name).toBe('f00000.txt');
|
||||
expect(children.at(-1)!.fsentry_name).toBe('f10049.txt');
|
||||
expect(new Set(children.map((c) => c.fsentry_name)).size).toBe(total);
|
||||
expect(children.some((c) => c.write_url)).toBe(false);
|
||||
}, 120_000);
|
||||
|
||||
it('answers a small folder in one body', async () => {
|
||||
const dir = await makeDir('small');
|
||||
await write(`${dir}/one.txt`, '1');
|
||||
const response = await signedListing(dir);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-length')).not.toBeNull();
|
||||
const children = (await response.json()) as Array<{
|
||||
fsentry_name: string;
|
||||
}>;
|
||||
expect(children.map((c) => c.fsentry_name)).toEqual(['one.txt']);
|
||||
});
|
||||
});
|
||||
@@ -153,34 +153,6 @@ describe('legacy file downloads over HTTP', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(routes)(
|
||||
'GET %s answers a ranged read of an empty file with an empty 200',
|
||||
async (route) => {
|
||||
const file = await writeFile(
|
||||
`empty-${route.replace(/\W/g, '')}.bin`,
|
||||
Buffer.alloc(0),
|
||||
);
|
||||
const ranged = await requestFor(route, file, {
|
||||
headers: { Range: 'bytes=0-511' },
|
||||
});
|
||||
expect(ranged.status).toBe(200);
|
||||
expect(await ranged.text()).toBe('');
|
||||
},
|
||||
);
|
||||
|
||||
it('GET /read answers byte_count on an empty file with an empty 200', async () => {
|
||||
const file = await writeFile('empty-byte-count.bin', Buffer.alloc(0));
|
||||
for (const query of ['byte_count=512', 'offset=0']) {
|
||||
const url = new URL(`/read?${query}`, env.apiOrigin);
|
||||
url.searchParams.set('file', file.path);
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${env.users.user.token}` },
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.text()).toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
// Storage hands over one chunk and then goes quiet, as a large object
|
||||
// does while the client is still reading.
|
||||
const stallReadsOf = (objectKey: string) => {
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
import type { Request, RequestHandler, Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { PuterRouter } from '../../core/http/PuterRouter.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { LegacyFSController } from './LegacyFSController.js';
|
||||
|
||||
const PNG_2X2 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEUlEQVR4nGP4z8DwH4QZYAwAR8oH+WdZbrcAAAAASUVORK5CYII=';
|
||||
|
||||
let server: PuterServer;
|
||||
let handler: RequestHandler;
|
||||
type Driver = {
|
||||
create: (a: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer({
|
||||
api_base_url: 'http://api.test.local',
|
||||
static_hosting_domain: 'site.test',
|
||||
} as unknown as Parameters<typeof setupTestServer>[0]);
|
||||
await server.services.appIcon.ensureIconsDirectory();
|
||||
const controller = server.controllers
|
||||
.legacyFs as unknown as LegacyFSController;
|
||||
const router = new PuterRouter();
|
||||
controller.registerRoutes(router);
|
||||
handler = router.routes.find(
|
||||
(r) =>
|
||||
r.path === '/get-launch-apps' &&
|
||||
String(r.method).toLowerCase() === 'get',
|
||||
)!.handler as RequestHandler;
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
const makeActor = async (): Promise<{ actor: Actor; id: number }> => {
|
||||
const username = `v13-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const u = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
const r = (await server.stores.user.getById(u.id))!;
|
||||
return {
|
||||
id: r.id,
|
||||
actor: {
|
||||
user: {
|
||||
id: r.id,
|
||||
uuid: r.uuid,
|
||||
username: r.username,
|
||||
email: r.email ?? null,
|
||||
email_confirmed: true,
|
||||
},
|
||||
} as unknown as Actor,
|
||||
};
|
||||
};
|
||||
|
||||
const createViaDriver = async (actor: Actor, icon: string) =>
|
||||
runWithContext({ actor }, () =>
|
||||
(server.drivers.apps as unknown as Driver).create({
|
||||
object: {
|
||||
name: `v13-${Math.random().toString(36).slice(2, 10)}`,
|
||||
title: 'V',
|
||||
index_url: `https://v13-${Math.random().toString(36).slice(2, 10)}.test/`,
|
||||
icon,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const launchApps = async (actor: Actor, appUid: string, userId: number) => {
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `app_opens` (`app_uid`, `user_id`, `ts`) VALUES (?, ?, ?)',
|
||||
[appUid, userId, Math.floor(Date.now() / 1000)],
|
||||
);
|
||||
let body: { recent: Array<Record<string, unknown>> } | undefined;
|
||||
const res = {
|
||||
json: (v: unknown) => {
|
||||
body = v as typeof body;
|
||||
return res;
|
||||
},
|
||||
status: () => res,
|
||||
setHeader: () => res,
|
||||
} as unknown as Response;
|
||||
await handler(
|
||||
{
|
||||
query: { icon_size: '64' },
|
||||
headers: {},
|
||||
body: {},
|
||||
actor,
|
||||
} as unknown as Request,
|
||||
res,
|
||||
() => {},
|
||||
);
|
||||
return body!.recent.find((e) => e.uuid === appUid)!;
|
||||
};
|
||||
|
||||
describe('/get-launch-apps recent icon, verified', () => {
|
||||
it('a valid data-URL icon written through the driver is migrated to a short URL', async () => {
|
||||
const { actor, id } = await makeActor();
|
||||
const app = await createViaDriver(
|
||||
actor,
|
||||
`data:image/png;base64,${PNG_2X2}`,
|
||||
);
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
const row = await server.stores.app.getByUid(app.uid as string);
|
||||
expect(String(row?.icon).slice(0, 80)).toMatch(
|
||||
/^https?:\/\/.*\/app-icon\/app-/,
|
||||
);
|
||||
},
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
const entry = await launchApps(actor, app.uid as string, id);
|
||||
expect(String(entry.icon).length).toBeLessThan(200);
|
||||
expect(entry.iconCdnUrl).toMatch(/-64\.png$/);
|
||||
}, 30_000);
|
||||
|
||||
it('returns an icon endpoint when migration cannot decode the stored icon', async () => {
|
||||
const { actor, id } = await makeActor();
|
||||
// PNG signature passes the write-path sniff; sharp rejects the rest.
|
||||
const corrupt = Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
Buffer.alloc(1024 * 1024, 0x41),
|
||||
]).toString('base64');
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
let app: Record<string, unknown>;
|
||||
try {
|
||||
app = await createViaDriver(
|
||||
actor,
|
||||
`data:image/png;base64,${corrupt}`,
|
||||
);
|
||||
// Give the background pipeline time to try and fail.
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(
|
||||
warn.mock.calls.some(
|
||||
([m]) => m === '[app-icon] icon processing failed',
|
||||
),
|
||||
).toBe(true),
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
const row = await server.stores.app.getByUid(app!.uid as string);
|
||||
expect(String(row?.icon).startsWith('data:image/png')).toBe(true);
|
||||
const entry = await launchApps(actor, app!.uid as string, id);
|
||||
expect(entry.icon).toBe(
|
||||
`http://api.test.local/app-icon/${app!.uid}/64`,
|
||||
);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -5293,39 +5293,4 @@ describe('LegacyFSController GET /get-launch-apps', () => {
|
||||
|
||||
expect(returned).toEqual(recentUids);
|
||||
});
|
||||
|
||||
it('skips the recent opens for recent=0 and keeps recommended', async () => {
|
||||
const { actor, userId } = await makeUser();
|
||||
const app = await makeHostedApp(userId, 'unregistered-skip');
|
||||
await recordOpen(userId, app.uid);
|
||||
|
||||
const call = async (query: Record<string, unknown>) => {
|
||||
const { res, captured } = makeRes();
|
||||
await withActor(actor, async () => {
|
||||
await getLaunchAppsHandler()(
|
||||
makeReq({ actor, query }),
|
||||
res,
|
||||
() => {
|
||||
throw new Error('handler called next() unexpectedly');
|
||||
},
|
||||
);
|
||||
});
|
||||
return captured.body as {
|
||||
recommended: unknown[];
|
||||
recent: unknown[];
|
||||
};
|
||||
};
|
||||
const full = await call({});
|
||||
expect(full.recent.length).toBeGreaterThan(0);
|
||||
|
||||
const opens = vi.spyOn(server.stores.app, 'getRecentAppOpens');
|
||||
try {
|
||||
const skipped = await call({ recent: '0' });
|
||||
expect(skipped.recent).toEqual([]);
|
||||
expect(skipped.recommended).toEqual(full.recommended);
|
||||
expect(opens).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
opens.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -26,7 +26,6 @@ import {
|
||||
assertResolvedActor,
|
||||
isAccessTokenActor,
|
||||
isAppActor,
|
||||
isPlainUserActor,
|
||||
makeActor,
|
||||
} from '../../core/actor.js';
|
||||
import { Context } from '../../core/context.js';
|
||||
@@ -96,7 +95,6 @@ import {
|
||||
assertAccess,
|
||||
assertCanCreate,
|
||||
assertCanMoveInto,
|
||||
contextBoundStream,
|
||||
emitGuiItemEvent,
|
||||
expandClientPath,
|
||||
findOverwriteTarget,
|
||||
@@ -104,7 +102,6 @@ import {
|
||||
getString,
|
||||
type GuiItemEventName,
|
||||
loadLegacyAssociatedApps,
|
||||
rangeForEntry,
|
||||
replacedByMove,
|
||||
resolveReadRange,
|
||||
resolveV1Selector,
|
||||
@@ -384,9 +381,7 @@ export class LegacyFSController extends PuterController {
|
||||
|
||||
let recent: unknown[] = [];
|
||||
const userId = req.actor?.user?.id;
|
||||
// `recent=0`: a caller that only shows `recommended`.
|
||||
const wantsRecent = req.query.recent !== '0';
|
||||
if (userId && wantsRecent) {
|
||||
if (userId) {
|
||||
const recentUids = await this.stores.app.getRecentAppOpens(
|
||||
userId,
|
||||
{ limit: 10 },
|
||||
@@ -423,9 +418,9 @@ export class LegacyFSController extends PuterController {
|
||||
return {
|
||||
...toAppSummary(app, {
|
||||
config: this.config,
|
||||
apiBaseUrl: this.config.api_base_url,
|
||||
iconSize,
|
||||
}),
|
||||
icon: app.icon ?? null,
|
||||
index_url: backingGone ? null : app.index_url,
|
||||
...(backingGone
|
||||
? { privateAccess: buildHostedBackingDenial() }
|
||||
@@ -806,23 +801,10 @@ export class LegacyFSController extends PuterController {
|
||||
this.stores.fsEntry,
|
||||
body.source,
|
||||
);
|
||||
const destination = await resolveV1Selector(
|
||||
const destinationParent = await resolveV1Selector(
|
||||
this.stores.fsEntry,
|
||||
body.destination,
|
||||
);
|
||||
// A file destination names the copy itself, replaced only with
|
||||
// `overwrite`.
|
||||
let newName = getString(body, 'new_name');
|
||||
let destinationParent = destination;
|
||||
if (!destination.isDir && !newName && destination.parentUid) {
|
||||
const parent = await this.stores.fsEntry.getEntryByUuid(
|
||||
destination.parentUid,
|
||||
);
|
||||
if (parent) {
|
||||
destinationParent = parent;
|
||||
newName = destination.name;
|
||||
}
|
||||
}
|
||||
|
||||
await assertAccess(
|
||||
this.services.acl,
|
||||
@@ -847,14 +829,14 @@ export class LegacyFSController extends PuterController {
|
||||
? await findOverwriteTarget(
|
||||
this.stores.fsEntry,
|
||||
destinationParent,
|
||||
newName ?? source.name,
|
||||
getString(body, 'new_name') ?? source.name,
|
||||
)
|
||||
: null;
|
||||
|
||||
const copy = await this.services.fs.copy(userId, {
|
||||
source,
|
||||
destinationParent,
|
||||
newName,
|
||||
newName: getString(body, 'new_name'),
|
||||
overwrite: overwriteRequested,
|
||||
dedupeName: getBoolean(body, 'dedupe_name', 'change_name') ?? false,
|
||||
});
|
||||
@@ -1307,10 +1289,7 @@ export class LegacyFSController extends PuterController {
|
||||
});
|
||||
}
|
||||
|
||||
const range = rangeForEntry(
|
||||
entry,
|
||||
resolveReadRange(req.headers.range, query),
|
||||
);
|
||||
const range = resolveReadRange(req.headers.range, query);
|
||||
const download = await this.services.fs.readContent(entry, {
|
||||
range,
|
||||
});
|
||||
@@ -1811,50 +1790,19 @@ export class LegacyFSController extends PuterController {
|
||||
// owner, so the layout it reveals is theirs to reveal, and the child
|
||||
// signatures expire; the paths are the only part that outlives them.
|
||||
if (entry.isDir) {
|
||||
const signChild = (child: FSEntry) => {
|
||||
const children = await this.services.fs.listDirectory(entry.uuid);
|
||||
const signedChildren = children.map((child) => {
|
||||
const { write_url: _, ...rest } = signEntry(child, signingCfg, {
|
||||
ttlSeconds: NON_OWNER_SIGNATURE_TTL_SECONDS,
|
||||
});
|
||||
return { ...rest, path: maskEntryPath(child) };
|
||||
};
|
||||
const pages = this.services.fs.listDirectoryPages(entry.uuid);
|
||||
const first = await pages.next();
|
||||
const firstPage = first.done ? [] : first.value;
|
||||
const second = await pages.next();
|
||||
if (second.done) {
|
||||
res.json(firstPage.map(signChild));
|
||||
return;
|
||||
}
|
||||
|
||||
// Past one page the listing streams, so a large folder isn't held
|
||||
// in memory whole. A failure partway cuts the response off.
|
||||
const secondPage = second.value;
|
||||
const serialize = (child: FSEntry) =>
|
||||
JSON.stringify(signChild(child));
|
||||
async function* allPages() {
|
||||
yield firstPage;
|
||||
yield secondPage;
|
||||
yield* pages;
|
||||
}
|
||||
async function* body() {
|
||||
let separator = '[';
|
||||
for await (const page of allPages()) {
|
||||
if (page.length === 0) continue;
|
||||
yield separator + page.map(serialize).join(',');
|
||||
separator = ',';
|
||||
}
|
||||
yield separator === '[' ? '[]' : ']';
|
||||
}
|
||||
res.type('application/json');
|
||||
await pipeToResponse(contextBoundStream(body()), res);
|
||||
});
|
||||
res.json(signedChildren);
|
||||
return;
|
||||
}
|
||||
|
||||
// File: stream bytes with Range support.
|
||||
const range = rangeForEntry(
|
||||
entry,
|
||||
resolveReadRange(req.headers.range, query),
|
||||
);
|
||||
const range = resolveReadRange(req.headers.range, query);
|
||||
const download = await this.services.fs.readContent(entry, {
|
||||
range,
|
||||
});
|
||||
@@ -2044,11 +1992,6 @@ export class LegacyFSController extends PuterController {
|
||||
*/
|
||||
checkAppAcl = async (req: Request, res: Response): Promise<void> => {
|
||||
const actor = this.#requireActor(req);
|
||||
// Sessions only: for an app, this would be a window onto its neighbours' grants.
|
||||
if (!isPlainUserActor(actor))
|
||||
throw new HttpError(403, 'actor must be a user', {
|
||||
legacyCode: 'forbidden',
|
||||
});
|
||||
const body = asRecord(req.body);
|
||||
|
||||
const subjectRef = body.subject;
|
||||
@@ -2079,7 +2022,7 @@ export class LegacyFSController extends PuterController {
|
||||
legacyCode: 'subject_does_not_exist',
|
||||
});
|
||||
|
||||
let app: { uid: string; id: number } | null = null;
|
||||
let app: { uid: string } | null = null;
|
||||
if (typeof appRef === 'string') {
|
||||
app =
|
||||
(await this.stores.app.getByUid(appRef)) ??
|
||||
@@ -2093,7 +2036,7 @@ export class LegacyFSController extends PuterController {
|
||||
// Build an actor-under-user shape for the check.
|
||||
const actorForApp = makeActor({
|
||||
user: actor.user,
|
||||
app: { uid: app.uid, id: app.id },
|
||||
app: { uid: (app as { uid: string }).uid },
|
||||
});
|
||||
const allowed = await acl.check(actorForApp, descriptor, mode);
|
||||
res.json({ allowed });
|
||||
@@ -2141,12 +2084,10 @@ export class LegacyFSController extends PuterController {
|
||||
// by default, so opt in by naming the downloader as who pays.
|
||||
req.egressActor = actor;
|
||||
|
||||
const range = rangeForEntry(
|
||||
entry,
|
||||
const range =
|
||||
typeof req.headers.range === 'string'
|
||||
? req.headers.range
|
||||
: undefined,
|
||||
);
|
||||
: undefined;
|
||||
const download = await this.services.fs.readContent(entry, {
|
||||
range,
|
||||
});
|
||||
|
||||
@@ -17,19 +17,15 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { Writable } from 'node:stream';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { Context, runWithContext } from '../../core/context.js';
|
||||
import type { PuterServer } from '../../server.js';
|
||||
import { createTestUser, setupTestServer } from '../../testUtil.js';
|
||||
import {
|
||||
asRecord,
|
||||
contextBoundStream,
|
||||
fsEntryMimeType,
|
||||
getBoolean,
|
||||
getString,
|
||||
loadLegacyAssociatedApps,
|
||||
rangeForEntry,
|
||||
resolveReadRange,
|
||||
signEntry,
|
||||
signEntryThumbnail,
|
||||
@@ -709,50 +705,3 @@ describe('resolveReadRange', () => {
|
||||
).toThrow('Invalid byte_count');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rangeForEntry', () => {
|
||||
it('drops the range for an empty file only', () => {
|
||||
const entry = (size: number) => ({ size }) as FSEntry;
|
||||
expect(rangeForEntry(entry(0), 'bytes=0-511')).toBeUndefined();
|
||||
expect(rangeForEntry(entry(10), 'bytes=0-4')).toBe('bytes=0-4');
|
||||
});
|
||||
});
|
||||
|
||||
describe('contextBoundStream', () => {
|
||||
it("reads every chunk in the caller's context", async () => {
|
||||
// A sink slower than the source that completes writes from outside
|
||||
// any request, so later chunks are asked for from its drain.
|
||||
const pending: Array<() => void> = [];
|
||||
const flush = runWithContext({}, () =>
|
||||
setInterval(() => pending.splice(0).forEach((done) => done()), 5),
|
||||
);
|
||||
const sink = new Writable({
|
||||
highWaterMark: 1,
|
||||
write(_chunk, _encoding, done) {
|
||||
pending.push(done);
|
||||
},
|
||||
});
|
||||
const seen: unknown[] = [];
|
||||
try {
|
||||
await runWithContext({ actor: 'caller' as never }, async () => {
|
||||
async function* chunks() {
|
||||
for (let i = 0; i < 40; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
seen.push(Context.get('actor'));
|
||||
yield 'x';
|
||||
}
|
||||
}
|
||||
await new Promise((resolve, reject) => {
|
||||
contextBoundStream(chunks())
|
||||
.pipe(sink)
|
||||
.on('finish', resolve)
|
||||
.on('error', reject);
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
clearInterval(flush);
|
||||
}
|
||||
expect(seen).toHaveLength(40);
|
||||
expect(seen.every((actor) => actor === 'caller')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -17,9 +17,7 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { AsyncResource } from 'node:async_hooks';
|
||||
import { posix as pathPosix } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { contentType as contentTypeFromMime } from 'mime-types';
|
||||
import type { FSEntry } from '../../stores/fs/FSEntry.js';
|
||||
import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js';
|
||||
@@ -152,29 +150,6 @@ export function resolveReadRange(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A response body read from `chunks` for `pipeToResponse`. Each chunk is pulled
|
||||
* in the caller's async context, which share masking reads, not in whichever
|
||||
* socket event asked for more.
|
||||
*/
|
||||
export function contextBoundStream(
|
||||
chunks: AsyncGenerator<string, void, undefined>,
|
||||
): Readable {
|
||||
const next = AsyncResource.bind(() => chunks.next());
|
||||
const stop = AsyncResource.bind(() => chunks.return(undefined));
|
||||
return Readable.from({
|
||||
[Symbol.asyncIterator]: () => ({ next, return: stop }),
|
||||
});
|
||||
}
|
||||
|
||||
/** No range is satisfiable on an empty file, so it reads whole: empty, 200. */
|
||||
export function rangeForEntry(
|
||||
entry: FSEntry,
|
||||
range: string | undefined,
|
||||
): string | undefined {
|
||||
return entry.size === 0 ? undefined : range;
|
||||
}
|
||||
|
||||
// Accepts either `{ path }` or `{ uid }` or `{ id }` or `{ parent, name }`
|
||||
// from a legacy body field. Returns a resolved entry (throwing 404 if not
|
||||
// found or 400 if no usable ref is present).
|
||||
|
||||
@@ -17,9 +17,6 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type { Request, RequestHandler, Response } from 'express';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
@@ -469,63 +466,3 @@ describe('HomepageController GET /show/*splat', () => {
|
||||
expect(html).toContain('/alice/Documents');
|
||||
});
|
||||
});
|
||||
|
||||
// ── /sw.js ──────────────────────────────────────────────────────────
|
||||
|
||||
describe('HomepageController GET /sw.js', () => {
|
||||
const headersOf = (res: Response): Record<string, string> =>
|
||||
Object.fromEntries(
|
||||
(res.set as ReturnType<typeof vi.fn>).mock.calls.map(
|
||||
([key, value]) => [String(key).toLowerCase(), String(value)],
|
||||
),
|
||||
);
|
||||
|
||||
it('serves the worker with root scope, as JavaScript, revalidated each time', async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), 'puter-sw-route-'));
|
||||
await mkdir(path.join(root, 'dist'));
|
||||
await writeFile(path.join(root, 'dist', 'sw.js'), '// worker\n');
|
||||
const config = server.services.homepage.config as {
|
||||
gui_assets_root?: string;
|
||||
env?: string;
|
||||
};
|
||||
const prev = { root: config.gui_assets_root, env: config.env };
|
||||
config.gui_assets_root = root;
|
||||
config.env = 'prod';
|
||||
const { res, captured } = makeRes();
|
||||
try {
|
||||
await callRoute('get', '/sw.js', makeReq({ path: '/sw.js' }), res);
|
||||
} finally {
|
||||
config.gui_assets_root = prev.root;
|
||||
config.env = prev.env;
|
||||
}
|
||||
expect(captured.statusCode).toBe(200);
|
||||
expect(captured.body).toBe('// worker\n');
|
||||
expect(headersOf(res)).toMatchObject({
|
||||
'content-type': 'text/javascript; charset=utf-8',
|
||||
'cache-control': 'no-cache',
|
||||
'service-worker-allowed': '/',
|
||||
});
|
||||
});
|
||||
|
||||
it('is a 404 when there is no worker to serve', async () => {
|
||||
const config = server.services.homepage.config as {
|
||||
gui_assets_root?: string;
|
||||
};
|
||||
const prev = config.gui_assets_root;
|
||||
config.gui_assets_root = path.join(tmpdir(), 'no-such-gui-root');
|
||||
const { res, captured } = makeRes();
|
||||
try {
|
||||
await callRoute('get', '/sw.js', makeReq({ path: '/sw.js' }), res);
|
||||
} finally {
|
||||
config.gui_assets_root = prev;
|
||||
}
|
||||
expect(captured.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('is registered for the root subdomain only', () => {
|
||||
const route = router.routes.find(
|
||||
(r) => r.method === 'get' && r.path === '/sw.js',
|
||||
);
|
||||
expect(route?.options.subdomain).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -198,20 +198,6 @@ export class HomepageController extends PuterController {
|
||||
return sendShell(req, res, {}, launch);
|
||||
});
|
||||
|
||||
// -- /sw.js - the service worker, scoped to the whole origin --
|
||||
|
||||
router.get('/sw.js', { subdomain: '' }, async (_req, res) => {
|
||||
const source = await homepage.serviceWorkerSource();
|
||||
if (source === null) {
|
||||
res.status(404).send('Not found');
|
||||
return;
|
||||
}
|
||||
res.set('Content-Type', 'text/javascript; charset=utf-8');
|
||||
res.set('Cache-Control', 'no-cache');
|
||||
res.set('Service-Worker-Allowed', '/');
|
||||
res.send(source);
|
||||
});
|
||||
|
||||
// -- Fallback static mount -----------------------------------
|
||||
// Serves lingering GUI assets (images, fonts, lib files, etc.)
|
||||
// out of <gui_assets_root>/src. Falls through to the 404 handler
|
||||
|
||||
@@ -198,7 +198,7 @@ describe('HostingController /delete-site', () => {
|
||||
);
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
'subdomain.delete',
|
||||
{ subdomain: site.subdomain, uid: String(site.uuid), userId },
|
||||
{ subdomain: site.subdomain, uid: String(site.uuid) },
|
||||
{},
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -42,7 +42,6 @@ import {
|
||||
EVENTS_KV_HANDLES_PER_USER,
|
||||
KV_SHARE_BATCH_MAX,
|
||||
KV_SHARE_GRANT_LIMITS,
|
||||
KV_SHARE_LOOKUP_LIMIT,
|
||||
} from '../events/limits.js';
|
||||
|
||||
const BOOT_TIMEOUT_MS = 120_000;
|
||||
@@ -68,7 +67,6 @@ const resetBudgets = async (env: PuterTestEnv) => {
|
||||
const user = await env.server.stores.user.getByUsername(name);
|
||||
for (const scope of [
|
||||
EVENTS_KV_HANDLE_LIMIT.scope,
|
||||
KV_SHARE_LOOKUP_LIMIT.scope,
|
||||
...KV_SHARE_GRANT_LIMITS.map((window) => window.scope),
|
||||
])
|
||||
await env.server.clients.redis.del(`rate:${scope}:${user!.id}`);
|
||||
@@ -125,7 +123,6 @@ describe('/kv/shares', () => {
|
||||
'handle',
|
||||
'holder',
|
||||
'label',
|
||||
'notify',
|
||||
'prefix',
|
||||
].sort(),
|
||||
);
|
||||
@@ -134,42 +131,6 @@ describe('/kv/shares', () => {
|
||||
appUid: null,
|
||||
holder: env.users.other.username,
|
||||
label: 'Groceries',
|
||||
notify: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('takes notify, keeps it on a re-share that leaves it out, and shows it to both sides', async () => {
|
||||
const prefix = `notify-${uuidv4().slice(0, 8)}:`;
|
||||
const on = await shareWithOther({ prefix, notify: true });
|
||||
expect(on).toMatchObject({ prefix, notify: true });
|
||||
|
||||
const kept = await shareWithOther({ prefix, access: 'write' });
|
||||
expect(kept).toMatchObject({ handle: on.handle, notify: true });
|
||||
|
||||
const listed = async (path: string, token: string) =>
|
||||
(
|
||||
(await (await req('GET', path, token)).json()) as {
|
||||
items: Array<Record<string, unknown>>;
|
||||
}
|
||||
).items.find((row) => row.handle === on.handle);
|
||||
expect(
|
||||
await listed('/kv/shares?limit=200', env.users.user.token),
|
||||
).toMatchObject({ notify: true });
|
||||
expect(
|
||||
await listed('/kv/shares/with-me?limit=200', env.users.other.token),
|
||||
).toMatchObject({ notify: true });
|
||||
|
||||
const off = await shareWithOther({ prefix, notify: false });
|
||||
expect(off).toMatchObject({ handle: on.handle, notify: false });
|
||||
|
||||
const refused = await req('POST', '/kv/shares', env.users.user.token, {
|
||||
prefix,
|
||||
recipient: env.users.other.username,
|
||||
notify: 'yes',
|
||||
});
|
||||
expect(refused.status).toBe(400);
|
||||
await expect(refused.json()).resolves.toMatchObject({
|
||||
code: 'kv_share_notify_invalid',
|
||||
});
|
||||
});
|
||||
|
||||
@@ -208,7 +169,6 @@ describe('/kv/shares', () => {
|
||||
appUid: null,
|
||||
holder: env.users.other.username,
|
||||
label: 'Trip',
|
||||
notify: false,
|
||||
createdAt: shared.createdAt,
|
||||
revokedAt: null,
|
||||
revokedBy: null,
|
||||
@@ -230,7 +190,6 @@ describe('/kv/shares', () => {
|
||||
access: 'write',
|
||||
appUid: null,
|
||||
label: 'Trip',
|
||||
notify: false,
|
||||
createdAt: shared.createdAt,
|
||||
});
|
||||
expect(withMeText).not.toContain(env.users.user.username);
|
||||
@@ -663,162 +622,6 @@ describe('/kv/shares', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('sharing again', () => {
|
||||
let ownerId: number;
|
||||
let third: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
third = `kv-again-${uuidv4().slice(0, 8)}`;
|
||||
await createTestUser(env.server, {
|
||||
username: third,
|
||||
password: 'pw-test-1234',
|
||||
});
|
||||
ownerId = (await env.server.stores.user.getByUsername(
|
||||
env.users.user.username,
|
||||
))!.id!;
|
||||
});
|
||||
|
||||
const shareAt = (prefix: string, body: Record<string, unknown>) =>
|
||||
req('POST', '/kv/shares', env.users.user.token, {
|
||||
prefix,
|
||||
...body,
|
||||
});
|
||||
|
||||
/** Leaves `left` units in one of the owner's windows. */
|
||||
const drain = (
|
||||
window: { scope: string; limit: number; window: number },
|
||||
left = 0,
|
||||
) =>
|
||||
claimRateLimit(
|
||||
`${window.scope}:${ownerId}`,
|
||||
window.limit,
|
||||
window.window,
|
||||
window.limit - left,
|
||||
);
|
||||
|
||||
it('spends neither the mint nor the grant budget when every share stays as it is', async () => {
|
||||
const prefix = `again-${uuidv4().slice(0, 8)}:`;
|
||||
const recipients = [env.users.other.username, third];
|
||||
const made = (await (
|
||||
await shareAt(prefix, { recipients, label: 'Room' })
|
||||
).json()) as { items: Array<{ handle: string }> };
|
||||
await drain(EVENTS_KV_HANDLE_LIMIT);
|
||||
for (const window of KV_SHARE_GRANT_LIMITS) await drain(window);
|
||||
try {
|
||||
const again = await shareAt(prefix, {
|
||||
recipients: [...recipients, third],
|
||||
});
|
||||
expect(again.status).toBe(200);
|
||||
const { items } = (await again.json()) as {
|
||||
items: Array<{ handle: string; label: string }>;
|
||||
};
|
||||
expect(items.map((item) => item.handle)).toEqual([
|
||||
made.items[0].handle,
|
||||
made.items[1].handle,
|
||||
made.items[1].handle,
|
||||
]);
|
||||
expect(items[0].label).toBe('Room');
|
||||
|
||||
const single = await shareAt(prefix, {
|
||||
recipient: env.users.other.username,
|
||||
access: 'read',
|
||||
});
|
||||
expect(single.status).toBe(200);
|
||||
} finally {
|
||||
await resetBudgets(env);
|
||||
}
|
||||
});
|
||||
|
||||
it('spends the mint budget on a change, and grant units only on recipients without the share', async () => {
|
||||
const prefix = `again-${uuidv4().slice(0, 8)}:`;
|
||||
const other = env.users.other.username;
|
||||
expect((await shareAt(prefix, { recipient: other })).status).toBe(
|
||||
200,
|
||||
);
|
||||
for (const window of KV_SHARE_GRANT_LIMITS) await drain(window);
|
||||
try {
|
||||
// A level change on a standing share takes no grant unit.
|
||||
const raised = await shareAt(prefix, {
|
||||
recipients: [other],
|
||||
access: 'write',
|
||||
});
|
||||
expect(raised.status).toBe(200);
|
||||
|
||||
// Naming an unknown user next to it costs what a new share
|
||||
// would, so the window refuses both.
|
||||
for (const extra of ['nobody-by-this-name', third]) {
|
||||
const refused = await shareAt(prefix, {
|
||||
recipients: [other, extra],
|
||||
access: 'write',
|
||||
});
|
||||
expect(refused.status).toBe(429);
|
||||
}
|
||||
} finally {
|
||||
await resetBudgets(env);
|
||||
}
|
||||
|
||||
await drain(EVENTS_KV_HANDLE_LIMIT);
|
||||
try {
|
||||
const lowered = await shareAt(prefix, {
|
||||
recipient: other,
|
||||
access: 'read',
|
||||
});
|
||||
expect(lowered.status).toBe(429);
|
||||
await expect(lowered.json()).resolves.toMatchObject({
|
||||
code: 'too_many_requests',
|
||||
});
|
||||
} finally {
|
||||
await resetBudgets(env);
|
||||
}
|
||||
});
|
||||
|
||||
it('counts turning notify on or off as a change', async () => {
|
||||
const prefix = `again-${uuidv4().slice(0, 8)}:`;
|
||||
const other = env.users.other.username;
|
||||
expect((await shareAt(prefix, { recipient: other })).status).toBe(
|
||||
200,
|
||||
);
|
||||
await drain(EVENTS_KV_HANDLE_LIMIT);
|
||||
try {
|
||||
expect(
|
||||
(await shareAt(prefix, { recipient: other })).status,
|
||||
).toBe(200);
|
||||
expect(
|
||||
(
|
||||
await shareAt(prefix, {
|
||||
recipient: other,
|
||||
notify: true,
|
||||
})
|
||||
).status,
|
||||
).toBe(429);
|
||||
} finally {
|
||||
await resetBudgets(env);
|
||||
}
|
||||
});
|
||||
|
||||
it('spends the lookup budget on every recipient named', async () => {
|
||||
const prefix = `again-${uuidv4().slice(0, 8)}:`;
|
||||
const other = env.users.other.username;
|
||||
expect((await shareAt(prefix, { recipient: other })).status).toBe(
|
||||
200,
|
||||
);
|
||||
await resetBudgets(env);
|
||||
await drain(KV_SHARE_LOOKUP_LIMIT, 1);
|
||||
try {
|
||||
expect(
|
||||
(await shareAt(prefix, { recipient: other })).status,
|
||||
).toBe(200);
|
||||
const refused = await shareAt(prefix, { recipient: other });
|
||||
expect(refused.status).toBe(429);
|
||||
await expect(refused.json()).resolves.toMatchObject({
|
||||
code: 'too_many_requests',
|
||||
});
|
||||
} finally {
|
||||
await resetBudgets(env);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('/events/kv-handles beside /kv/shares', () => {
|
||||
|
||||
@@ -69,7 +69,6 @@ const shareBody = (share: KvShareView) => ({
|
||||
appUid: publicAppUid(share.appUid),
|
||||
holder: share.holderUsername,
|
||||
label: share.label,
|
||||
notify: share.notify,
|
||||
createdAt: share.createdAt,
|
||||
});
|
||||
|
||||
@@ -129,8 +128,8 @@ export class KvShareController extends PuterController {
|
||||
/**
|
||||
* POST /kv/shares — share a region with another account, `read` unless the
|
||||
* body asks otherwise. Sharing the same region with the same account again
|
||||
* changes its level, label or `notify` flag in place. `recipients` shares
|
||||
* it with several at once and reports each one on its own item.
|
||||
* changes its level or label in place. `recipients` shares it with several
|
||||
* at once and reports each one on its own item.
|
||||
*/
|
||||
@Post('/shares', {
|
||||
subdomain: 'api',
|
||||
@@ -146,7 +145,6 @@ export class KvShareController extends PuterController {
|
||||
access: body.access,
|
||||
appUid: body.appUid,
|
||||
label: body.label,
|
||||
notify: body.notify,
|
||||
};
|
||||
const recipients = batchRecipients(body);
|
||||
if (recipients) {
|
||||
@@ -208,7 +206,6 @@ export class KvShareController extends PuterController {
|
||||
appUid: publicAppUid(row.appUid),
|
||||
holder: row.holderUsername,
|
||||
label: row.label,
|
||||
notify: row.notify,
|
||||
createdAt: row.createdAt,
|
||||
revokedAt: row.revokedAt,
|
||||
revokedBy: row.revokedBy,
|
||||
@@ -237,7 +234,6 @@ export class KvShareController extends PuterController {
|
||||
access: row.access,
|
||||
appUid: publicAppUid(row.appUid),
|
||||
label: row.label,
|
||||
notify: row.notify,
|
||||
createdAt: row.createdAt,
|
||||
})),
|
||||
...(page.cursor ? { cursor: page.cursor } : {}),
|
||||
|
||||
@@ -21,9 +21,9 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
/**
|
||||
* Route-level coverage for the notification endpoints the desktop calls over
|
||||
* HTTP, so this suite is what catches a route going missing or a gate turning
|
||||
* a legitimate dismiss or removal away.
|
||||
* Route-level coverage for the notification endpoint. Dismissing is the only
|
||||
* mark the desktop performs over HTTP, so this suite is what catches the route
|
||||
* going missing or a gate turning a legitimate dismiss away.
|
||||
*/
|
||||
describe('notification endpoints over HTTP', () => {
|
||||
let env: PuterTestEnv;
|
||||
@@ -85,34 +85,4 @@ describe('notification endpoints over HTTP', () => {
|
||||
});
|
||||
expect(after?.acknowledged).toBeFalsy();
|
||||
});
|
||||
|
||||
it('removes a notification', async () => {
|
||||
const owner = env.users.user;
|
||||
const { userId, uid } = await seedNotification(owner.username);
|
||||
|
||||
const res = await post('/notif/delete', owner.token, { uid });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
await expect(res.json()).resolves.toEqual({});
|
||||
const after = await env.server.stores.notification.getByUid(uid, {
|
||||
userId,
|
||||
});
|
||||
expect(after).toBeNull();
|
||||
});
|
||||
|
||||
it('refuses an anonymous removal', async () => {
|
||||
const { userId, uid } = await seedNotification(env.users.other.username);
|
||||
|
||||
const res = await fetch(new URL('/notif/delete', env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ uid }),
|
||||
});
|
||||
|
||||
expect(res.status).toBeGreaterThanOrEqual(400);
|
||||
const after = await env.server.stores.notification.getByUid(uid, {
|
||||
userId,
|
||||
});
|
||||
expect(after?.uid).toBe(uid);
|
||||
});
|
||||
});
|
||||
@@ -183,74 +183,3 @@ describe('NotificationController.markAck', () => {
|
||||
expect(after?.acknowledged).toBeFalsy();
|
||||
});
|
||||
});
|
||||
|
||||
// ── /notif/delete ───────────────────────────────────────────────────
|
||||
|
||||
describe('NotificationController.remove', () => {
|
||||
it('deletes the underlying notification row', async () => {
|
||||
const { actor, userId } = await makeUser();
|
||||
const created = await server.stores.notification.create({
|
||||
userId,
|
||||
value: { title: 't' },
|
||||
});
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await controller.remove(
|
||||
makeReq({ body: { uid: created.uid }, actor }),
|
||||
res,
|
||||
);
|
||||
|
||||
expect(captured.body).toEqual({});
|
||||
const after = await server.stores.notification.getByUid(
|
||||
created.uid as string,
|
||||
{ userId },
|
||||
);
|
||||
expect(after).toBeNull();
|
||||
});
|
||||
|
||||
it('succeeds for a uid that names no row', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const { res, captured } = makeRes();
|
||||
await controller.remove(
|
||||
makeReq({ body: { uid: 'no-such-uid' }, actor }),
|
||||
res,
|
||||
);
|
||||
expect(captured.body).toEqual({});
|
||||
});
|
||||
|
||||
it('rejects a missing uid with 400', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
controller.remove(makeReq({ body: {}, actor }), res),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('throws 401 when there is no actor on the request', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
controller.remove(makeReq({ body: { uid: 'whatever' } }), res),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it("does not delete another user's notification", async () => {
|
||||
const a = await makeUser();
|
||||
const b = await makeUser();
|
||||
const created = await server.stores.notification.create({
|
||||
userId: a.userId,
|
||||
value: {},
|
||||
});
|
||||
|
||||
const { res } = makeRes();
|
||||
await controller.remove(
|
||||
makeReq({ body: { uid: created.uid }, actor: b.actor }),
|
||||
res,
|
||||
);
|
||||
|
||||
const after = await server.stores.notification.getByUid(
|
||||
created.uid as string,
|
||||
{ userId: a.userId },
|
||||
);
|
||||
expect(after?.uid).toBe(created.uid);
|
||||
});
|
||||
});
|
||||
@@ -20,44 +20,16 @@
|
||||
import type { Request, Response } from 'express';
|
||||
import { Controller, Post } from '../../core/http/decorators.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import type { NotificationService } from '../../services/notification/NotificationService.js';
|
||||
import { PuterController } from '../types.js';
|
||||
|
||||
/** Both routes fire per interaction; the ceiling catches a loop, not a user. */
|
||||
const MARK_ROUTE_OPTIONS = {
|
||||
subdomain: 'api',
|
||||
requireUserActor: true,
|
||||
allowFullAccessToken: true,
|
||||
rateLimit: {
|
||||
scope: 'notification-mark',
|
||||
limit: 300,
|
||||
window: 60_000,
|
||||
key: 'user',
|
||||
},
|
||||
} as const;
|
||||
|
||||
/** The uid and owner a mark names, or the 400/401 the request earns. */
|
||||
const markTarget = (req: Request): { uid: string; userId: number } => {
|
||||
const uid = req.body?.uid;
|
||||
if (typeof uid !== 'string' || uid.length === 0) {
|
||||
throw new HttpError(400, '`uid` must be a non-empty string', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
const userId = req.actor?.user?.id;
|
||||
if (!userId)
|
||||
throw new HttpError(401, 'Unauthorized', {
|
||||
legacyCode: 'unauthorized',
|
||||
});
|
||||
return { uid, userId };
|
||||
};
|
||||
|
||||
/**
|
||||
* GUI-facing notification endpoints. Supplement the `puter-notifications`
|
||||
* driver (which handles reads via `/drivers/call`) with the mutations the puter
|
||||
* desktop client calls directly.
|
||||
* GUI-facing notification endpoint. Supplements the `puter-notifications`
|
||||
* driver (which handles CRUD via `/drivers/call`) with the one mutation route
|
||||
* the puter desktop client calls directly.
|
||||
*
|
||||
* Each goes through the NotificationService, which tells the user's other open
|
||||
* tabs (`notif.ack`, `notif.removed`) so they update immediately.
|
||||
* It emits `outer.gui.notif.ack` via the NotificationService so other open tabs
|
||||
* for the same user see the state change immediately.
|
||||
*/
|
||||
@Controller('/notif')
|
||||
export class NotificationController extends PuterController {
|
||||
@@ -65,21 +37,50 @@ export class NotificationController extends PuterController {
|
||||
* POST /notif/mark-ack — user dismissed a notification. Sets `acknowledged`
|
||||
* timestamp; pushes ack event to sockets.
|
||||
*/
|
||||
@Post('/mark-ack', MARK_ROUTE_OPTIONS)
|
||||
@Post('/mark-ack', {
|
||||
subdomain: 'api',
|
||||
requireUserActor: true,
|
||||
allowFullAccessToken: true,
|
||||
// Fires per notification interaction, so the ceiling stays
|
||||
// generous — it is here to catch a loop, not to pace a user.
|
||||
rateLimit: {
|
||||
scope: 'notification-mark',
|
||||
limit: 300,
|
||||
window: 60_000,
|
||||
key: 'user',
|
||||
},
|
||||
})
|
||||
async markAck(req: Request, res: Response): Promise<void> {
|
||||
const { uid, userId } = markTarget(req);
|
||||
await this.services.notification.markAcknowledged(uid, userId);
|
||||
res.json({});
|
||||
}
|
||||
const uid = req.body?.uid;
|
||||
if (typeof uid !== 'string' || uid.length === 0) {
|
||||
throw new HttpError(400, '`uid` must be a non-empty string', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
const userId = req.actor?.user?.id;
|
||||
if (!userId)
|
||||
throw new HttpError(401, 'Unauthorized', {
|
||||
legacyCode: 'unauthorized',
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /notif/delete — user removed a notification from their history. A
|
||||
* uid naming none of their rows succeeds too: it is gone either way.
|
||||
*/
|
||||
@Post('/delete', MARK_ROUTE_OPTIONS)
|
||||
async remove(req: Request, res: Response): Promise<void> {
|
||||
const { uid, userId } = markTarget(req);
|
||||
await this.services.notification.remove(uid, userId);
|
||||
const notifService = this.services.notification as unknown as
|
||||
| NotificationService
|
||||
| undefined;
|
||||
if (notifService?.markAcknowledged) {
|
||||
await notifService.markAcknowledged(uid, userId);
|
||||
} else {
|
||||
// Fallback: direct store call if service isn't wired
|
||||
await (
|
||||
this.stores as Record<string, unknown> as {
|
||||
notification: {
|
||||
markAcknowledged: (
|
||||
uid: string,
|
||||
userId: number,
|
||||
) => Promise<boolean>;
|
||||
};
|
||||
}
|
||||
).notification.markAcknowledged(uid, userId);
|
||||
}
|
||||
res.json({});
|
||||
}
|
||||
}
|
||||
@@ -511,31 +511,6 @@ describe('PuterAIController.openaiChatCompletions', () => {
|
||||
|
||||
const completeArgs = completeSpy.mock.calls[0]![0];
|
||||
expect(completeArgs.provider).toBe('openai-completion');
|
||||
expect(completeArgs.fallback).toBeUndefined();
|
||||
});
|
||||
|
||||
it('forwards fallback: false so the driver stops at the first route', async () => {
|
||||
const completeSpy = stubChatComplete({
|
||||
message: { role: 'assistant', content: 'hi there' },
|
||||
finish_reason: 'stop',
|
||||
usage: { prompt_tokens: 4, completion_tokens: 2 },
|
||||
});
|
||||
|
||||
const { res } = makeRes();
|
||||
await controller.openaiChatCompletions(
|
||||
makeReq({
|
||||
body: {
|
||||
model: 'gpt-test',
|
||||
provider: 'openai-completion',
|
||||
fallback: false,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
},
|
||||
actor: makeUserActor(),
|
||||
}),
|
||||
res,
|
||||
);
|
||||
|
||||
expect(completeSpy.mock.calls[0]![0].fallback).toBe(false);
|
||||
});
|
||||
|
||||
it('still pins OpenAI when the caller names no model, so the default model does not move', async () => {
|
||||
|
||||
@@ -448,7 +448,6 @@ export class PuterAIController extends PuterController {
|
||||
}
|
||||
: {}),
|
||||
...openaiCompatProvider(body),
|
||||
...(body.fallback === false ? { fallback: false } : {}),
|
||||
};
|
||||
|
||||
const result = await this.#complete(res, completeArgs);
|
||||
@@ -546,7 +545,8 @@ export class PuterAIController extends PuterController {
|
||||
}
|
||||
} else if (ev.type === 'usage') {
|
||||
usageDetails = ev.usageDetails as
|
||||
UsageDetails | undefined;
|
||||
| UsageDetails
|
||||
| undefined;
|
||||
finishReason = ev.finish_reason as string | undefined;
|
||||
} else if (ev.type === 'error') {
|
||||
res.write(
|
||||
@@ -649,7 +649,6 @@ export class PuterAIController extends PuterController {
|
||||
: {}),
|
||||
...(finiteMaxTokens(body.max_tokens) ?? {}),
|
||||
...openaiCompatProvider(body),
|
||||
...(body.fallback === false ? { fallback: false } : {}),
|
||||
};
|
||||
|
||||
const completionId = `cmpl-${randomId()}`;
|
||||
@@ -695,7 +694,8 @@ export class PuterAIController extends PuterController {
|
||||
sendChunk(ev.text);
|
||||
} else if (ev.type === 'usage') {
|
||||
usageDetails = ev.usageDetails as
|
||||
UsageDetails | undefined;
|
||||
| UsageDetails
|
||||
| undefined;
|
||||
finishReason = ev.finish_reason as string | undefined;
|
||||
} else if (ev.type === 'error') {
|
||||
res.write(
|
||||
@@ -749,7 +749,8 @@ export class PuterAIController extends PuterController {
|
||||
text: extractTextContent(
|
||||
(
|
||||
messageResult.message as
|
||||
Record<string, unknown> | undefined
|
||||
| Record<string, unknown>
|
||||
| undefined
|
||||
)?.content,
|
||||
),
|
||||
index: 0,
|
||||
@@ -868,7 +869,6 @@ export class PuterAIController extends PuterController {
|
||||
}
|
||||
: {}),
|
||||
provider: providerName,
|
||||
...(body.fallback === false ? { fallback: false } : {}),
|
||||
};
|
||||
|
||||
const responseId = generateId('resp');
|
||||
|
||||
@@ -324,18 +324,6 @@ describe('parseAnthropicRequest', () => {
|
||||
expect(args.provider).toBe('claude');
|
||||
expect(args.normalize).toBe(false);
|
||||
expect(args.streamToolInput).toBe(true);
|
||||
expect(args.fallback).toBeUndefined();
|
||||
});
|
||||
|
||||
it('forwards fallback: false and nothing else for fallback', () => {
|
||||
expect(
|
||||
parseAnthropicRequest({ messages: [], fallback: false }, headers)
|
||||
.fallback,
|
||||
).toBe(false);
|
||||
expect(
|
||||
parseAnthropicRequest({ messages: [], fallback: 'no' }, headers)
|
||||
.fallback,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('maps thinking, tool_choice, stop_sequences, top_p/top_k', () => {
|
||||
|
||||
@@ -431,7 +431,6 @@ export const parseAnthropicRequest = (
|
||||
: {}),
|
||||
...(betas.length ? { anthropicBetas: betas } : {}),
|
||||
provider: toStringOrEmpty(body.provider) || 'claude',
|
||||
...(body.fallback === false ? { fallback: false } : {}),
|
||||
};
|
||||
// Silently ignored: metadata, service_tier, inference_geo, container,
|
||||
// fallbacks, diagnostics.
|
||||
|
||||
@@ -151,12 +151,6 @@ describe('StaticAssetsController client_libs_root', () => {
|
||||
|
||||
const sentPutility = await callGetHandler(router, '/putility/v1');
|
||||
expect(sentPutility).toEqual({ filename: 'putility.js/v1.js', root });
|
||||
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
expect(findRoute(router, 'get', route)?.options.subdomain).toBe('js');
|
||||
const sentWorker = await callGetHandler(router, route);
|
||||
expect(sentWorker).toEqual({ filename: 'puter.js/push-sw.js', root });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not register the client_libs routes when the root is unset', () => {
|
||||
@@ -192,11 +186,6 @@ describe('StaticAssetsController puterjs_root', () => {
|
||||
expect(v1.root).toBe(root);
|
||||
const bareV1 = await callGetHandler(router, '/v1');
|
||||
expect(bareV1.filename).toBe('puter.js');
|
||||
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
const worker = await callGetHandler(router, route);
|
||||
expect(worker).toEqual({ filename: 'push-sw.js', root });
|
||||
}
|
||||
});
|
||||
|
||||
it('serves puter.dev.js when present (dev webpack build)', async () => {
|
||||
|
||||
@@ -26,9 +26,9 @@ import type { PuterRouter } from '../../core/http/PuterRouter';
|
||||
/**
|
||||
* Static asset routes.
|
||||
*
|
||||
* /puter.js/v1, /puter.js/v2 → any subdomain /v1, /v2, /push-sw.js,
|
||||
* /putility/v1 → js subdomain /sdk/* → root subdomain — puter-js bundle
|
||||
* /dist/_, /src/_, /assets/* → root subdomain
|
||||
* /puter.js/v1, /puter.js/v2 → any subdomain /v1, /v2, /putility/v1 → js
|
||||
* subdomain /sdk/* → root subdomain — puter-js bundle /dist/_, /src/_,
|
||||
* /assets/* → root subdomain
|
||||
*
|
||||
* Each block depends on its config root (`client_libs_root`, `gui_assets_root`,
|
||||
* `puterjs_root`). When unset, that block is skipped — deployments that don't
|
||||
@@ -55,12 +55,6 @@ export class StaticAssetsController extends PuterController {
|
||||
router.get('/putility/v1', { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile('putility.js/v1.js', { root });
|
||||
});
|
||||
// `/v2/push-sw.js` is the path older docs gave.
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
router.get(route, { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile('puter.js/push-sw.js', { root });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// puter-js SDK mount. GUI loads it at `/sdk/puter.dev.js`; the
|
||||
@@ -106,11 +100,6 @@ export class StaticAssetsController extends PuterController {
|
||||
router.get('/v2', { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile(puterJsFile, { root: puterjsRoot });
|
||||
});
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
router.get(route, { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile('push-sw.js', { root: puterjsRoot });
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import {
|
||||
setupPuterTestEnv,
|
||||
TEST_OTHER_USER_CREDENTIALS,
|
||||
TEST_USER_CREDENTIALS,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
|
||||
/** PROPFIND has no paging, so a folder must list whole however large. */
|
||||
describe('PROPFIND on a folder larger than one query', () => {
|
||||
let env: PuterTestEnv;
|
||||
const total = 10_050;
|
||||
let dir: string;
|
||||
let dirUuid: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
const { username } = env.users.user;
|
||||
dir = `/${username}/Documents/big-${randomUUID().slice(0, 8)}`;
|
||||
const owner = (await env.server.stores.user.getByUsername(username))!;
|
||||
const parent = (await env.server.stores.fsEntry.getEntryByPath(
|
||||
`/${username}/Documents`,
|
||||
))!;
|
||||
dirUuid = randomUUID();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const insert = (rows: unknown[][]) =>
|
||||
env.server.clients.db.write(
|
||||
`INSERT INTO fsentries (uuid, user_id, parent_id, parent_uid, name, path, is_dir, modified, created, accessed, size) VALUES ${rows.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
|
||||
rows.flat(),
|
||||
);
|
||||
await insert([
|
||||
[
|
||||
dirUuid,
|
||||
owner.id,
|
||||
parent.id,
|
||||
parent.uuid,
|
||||
dir.split('/').at(-1),
|
||||
dir,
|
||||
1,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
0,
|
||||
],
|
||||
]);
|
||||
const folder = (await env.server.stores.fsEntry.getEntryByPath(dir))!;
|
||||
for (let start = 0; start < total; start += 500) {
|
||||
const rows: unknown[][] = [];
|
||||
for (let i = start; i < Math.min(total, start + 500); i++) {
|
||||
const name = `f${String(i).padStart(5, '0')}.txt`;
|
||||
rows.push([
|
||||
randomUUID(),
|
||||
owner.id,
|
||||
folder.id,
|
||||
folder.uuid,
|
||||
name,
|
||||
`${dir}/${name}`,
|
||||
0,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
0,
|
||||
]);
|
||||
}
|
||||
await insert(rows);
|
||||
}
|
||||
|
||||
const actor = { user: owner };
|
||||
await runWithContext({ actor: actor as never }, () =>
|
||||
env.server.services.share.share(actor as never, {
|
||||
uid: dirUuid,
|
||||
recipient: { username: env.users.other.username },
|
||||
mode: 'read',
|
||||
}),
|
||||
);
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const propfind = async (
|
||||
path: string,
|
||||
credentials: { username: string; password: string },
|
||||
) => {
|
||||
const port = new URL(env.apiOrigin).port;
|
||||
const auth = Buffer.from(
|
||||
`${credentials.username}:${credentials.password}`,
|
||||
).toString('base64');
|
||||
const response = await fetch(
|
||||
`http://dav.puter.localhost:${port}${encodeURI(path)}`,
|
||||
{
|
||||
method: 'PROPFIND',
|
||||
headers: { Authorization: `Basic ${auth}`, Depth: '1' },
|
||||
},
|
||||
);
|
||||
return { status: response.status, body: await response.text() };
|
||||
};
|
||||
|
||||
const hrefsOf = (xml: string) =>
|
||||
[...xml.matchAll(/<D:href>([^<]*)<\/D:href>/g)].map((m) => m[1]!);
|
||||
|
||||
it('lists every child', async () => {
|
||||
const { status, body } = await propfind(dir, TEST_USER_CREDENTIALS);
|
||||
expect(status).toBe(207);
|
||||
expect(body.endsWith('</D:multistatus>')).toBe(true);
|
||||
const hrefs = hrefsOf(body);
|
||||
expect(hrefs).toHaveLength(1 + total);
|
||||
expect(new Set(hrefs).size).toBe(1 + total);
|
||||
expect(body).toContain(`${dir}/f10049.txt`);
|
||||
}, 60_000);
|
||||
|
||||
it("masks every page of a share recipient's listing", async () => {
|
||||
const masked = `/${env.users.user.username}/${dirUuid}/${dir.split('/').at(-1)}`;
|
||||
const { status, body } = await propfind(
|
||||
masked,
|
||||
TEST_OTHER_USER_CREDENTIALS,
|
||||
);
|
||||
expect(status).toBe(207);
|
||||
const hrefs = hrefsOf(body);
|
||||
expect(hrefs).toHaveLength(1 + total);
|
||||
expect(hrefs.every((href) => href.startsWith(masked))).toBe(true);
|
||||
expect(body).not.toContain('/Documents/');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -24,7 +24,6 @@ import { posix as pathPosix } from 'node:path';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { pipeToResponse } from '../../core/http/pipeToResponse.js';
|
||||
import {
|
||||
assertNotSuspended,
|
||||
assertTeam2fa,
|
||||
@@ -59,7 +58,6 @@ import {
|
||||
import { Context } from '../../core/context.js';
|
||||
import type { FSEntry } from '../../stores/fs/FSEntry.js';
|
||||
import {
|
||||
contextBoundStream,
|
||||
emitGuiItemEvent,
|
||||
type GuiItemEventName,
|
||||
} from '../fs/legacyFsHelpers.js';
|
||||
@@ -602,41 +600,19 @@ export class WebDAVController extends PuterController {
|
||||
propfindEntry(maskEntryPath(entry), entry, entry.isDir),
|
||||
];
|
||||
|
||||
if (depth === '0' || !entry.isDir) {
|
||||
this.#sendMultistatus(res, responses);
|
||||
return;
|
||||
}
|
||||
|
||||
const toXml = (children: FSEntry[]) =>
|
||||
children
|
||||
.map((child) =>
|
||||
if (depth !== '0' && entry.isDir) {
|
||||
const children = await this.services.fs.listDirectory(
|
||||
entry.uuid,
|
||||
{},
|
||||
);
|
||||
for (const child of children) {
|
||||
responses.push(
|
||||
propfindEntry(maskEntryPath(child), child, child.isDir),
|
||||
)
|
||||
.join('\n');
|
||||
const pages = this.services.fs.listDirectoryPages(entry.uuid);
|
||||
const first = await pages.next();
|
||||
if (!first.done && first.value.length > 0)
|
||||
responses.push(toXml(first.value));
|
||||
const second = await pages.next();
|
||||
if (second.done) {
|
||||
this.#sendMultistatus(res, responses);
|
||||
return;
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Past one page the listing streams, so a large folder isn't held in
|
||||
// memory whole. A failure partway cuts the response off.
|
||||
const secondPage = second.value;
|
||||
async function* body() {
|
||||
yield `${MULTISTATUS_OPEN}${responses.join('\n')}`;
|
||||
yield `\n${toXml(secondPage)}`;
|
||||
for await (const page of pages)
|
||||
if (page.length > 0) yield `\n${toXml(page)}`;
|
||||
yield MULTISTATUS_CLOSE;
|
||||
}
|
||||
res.status(207).set({
|
||||
'Content-Type': 'application/xml; charset=utf-8',
|
||||
});
|
||||
await pipeToResponse(contextBoundStream(body()), res);
|
||||
this.#sendMultistatus(res, responses);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1313,12 +1289,8 @@ function escapeXml(text: string): string {
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
const MULTISTATUS_OPEN =
|
||||
'<?xml version="1.0" encoding="utf-8"?>\n<D:multistatus xmlns:D="DAV:">\n';
|
||||
const MULTISTATUS_CLOSE = '\n</D:multistatus>';
|
||||
|
||||
function wrapMultistatus(inner: string): string {
|
||||
return `${MULTISTATUS_OPEN}${inner}${MULTISTATUS_CLOSE}`;
|
||||
return `<?xml version="1.0" encoding="utf-8"?>\n<D:multistatus xmlns:D="DAV:">\n${inner}\n</D:multistatus>`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -63,17 +63,10 @@ export interface KnownContextFields {
|
||||
driverName: string;
|
||||
/**
|
||||
* Aborts when the client disconnects before the response has finished (set
|
||||
* by DriverController). Some drivers (Gemini, Replicate) stop early on it.
|
||||
* Async video and image jobs keep polling and are metered after a
|
||||
* disconnect, since the provider bills them either way.
|
||||
* by DriverController). Long-running drivers poll it so work nobody will
|
||||
* receive stops early and is never metered.
|
||||
*/
|
||||
abortSignal: AbortSignal;
|
||||
/**
|
||||
* Keeps the call's concurrency slot until `work` settles (set by
|
||||
* DriverController when the method is capped), for work that outlives the
|
||||
* response.
|
||||
*/
|
||||
keepDriverSlot: (work: Promise<unknown>) => void;
|
||||
/**
|
||||
* Aborts the chat driver's upstream request. A client disconnect doesn't
|
||||
* fire it: the upstream bills a request it has accepted either way, so the
|
||||
@@ -87,11 +80,6 @@ export interface KnownContextFields {
|
||||
* different provider's translation of it.
|
||||
*/
|
||||
strictUpstreamErrors: boolean;
|
||||
/**
|
||||
* Set while Puter files something into an account for it (mail delivery),
|
||||
* so filesystem events never report that write as the request actor's own.
|
||||
*/
|
||||
platformWrite: boolean;
|
||||
}
|
||||
|
||||
// Every key of `KnownContextFields`, checked both ways by the compiler, so
|
||||
@@ -102,10 +90,8 @@ const KNOWN_KEY_FLAGS: Record<keyof KnownContextFields, true> = {
|
||||
requestId: true,
|
||||
driverName: true,
|
||||
abortSignal: true,
|
||||
keepDriverSlot: true,
|
||||
upstreamSignal: true,
|
||||
strictUpstreamErrors: true,
|
||||
platformWrite: true,
|
||||
};
|
||||
const KNOWN_KEYS: ReadonlySet<string> = new Set(Object.keys(KNOWN_KEY_FLAGS));
|
||||
|
||||
|
||||
@@ -836,34 +836,6 @@ describe('requireVerifiedAccount', () => {
|
||||
expectHttpError(got, 403, 'card_verification_required');
|
||||
});
|
||||
|
||||
it('asks an email-gate-last account for card, then phone, then email', () => {
|
||||
const pending = {
|
||||
uuid: 'u-1',
|
||||
requires_email_confirmation: true,
|
||||
email_confirmed: false,
|
||||
requires_phone_verification: true,
|
||||
requires_card_verification: true,
|
||||
metadata: { emailGateLast: true },
|
||||
};
|
||||
const gateFor = (user: Record<string, unknown>) =>
|
||||
runGate(requireVerifiedAccount(), { actor: { user } });
|
||||
expectHttpError(gateFor(pending), 403, 'card_verification_required');
|
||||
expectHttpError(
|
||||
gateFor({ ...pending, requires_card_verification: false }),
|
||||
403,
|
||||
'phone_verification_required',
|
||||
);
|
||||
expectHttpError(
|
||||
gateFor({
|
||||
...pending,
|
||||
requires_card_verification: false,
|
||||
requires_phone_verification: false,
|
||||
}),
|
||||
403,
|
||||
'email_confirmation_required',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns 403 two_factor_required for a seat whose team requires 2FA', async () => {
|
||||
const teams = { getOrgSeat: async () => ({ require_2fa: 1 }) };
|
||||
const got = await runGateAsync(requireTeam2fa(teams), {
|
||||
|
||||
@@ -32,7 +32,6 @@ import {
|
||||
type CardFallbackDeps,
|
||||
} from '../../../util/cardFallback';
|
||||
import { reserveUsernames } from '../../../util/reservedUsernames';
|
||||
import { isEmailGateLast } from '../../../util/verificationOrder';
|
||||
import {
|
||||
isAccountContext,
|
||||
isAppActor,
|
||||
@@ -310,25 +309,11 @@ export const requireVerifiedAccount = (): RequestHandler =>
|
||||
export const assertVerifiedAccount = (
|
||||
user: AccountGateUser | undefined,
|
||||
): void => {
|
||||
const assertEmail = () => {
|
||||
if (user?.requires_email_confirmation && !user?.email_confirmed) {
|
||||
throw new HttpError(403, 'Please confirm your email to continue', {
|
||||
legacyCode: 'email_confirmation_required',
|
||||
});
|
||||
}
|
||||
};
|
||||
const assertCard = () => {
|
||||
if (user?.requires_card_verification) {
|
||||
throw new HttpError(403, 'Please verify your card to continue', {
|
||||
legacyCode: 'card_verification_required',
|
||||
});
|
||||
}
|
||||
};
|
||||
// Email → phone → card, or card → phone → email for an email-gate-last
|
||||
// account, so the client is sent to the gate it has to clear next.
|
||||
const emailLast = isEmailGateLast(user);
|
||||
if (emailLast) assertCard();
|
||||
else assertEmail();
|
||||
if (user?.requires_email_confirmation && !user?.email_confirmed) {
|
||||
throw new HttpError(403, 'Please confirm your email to continue', {
|
||||
legacyCode: 'email_confirmation_required',
|
||||
});
|
||||
}
|
||||
if (user?.requires_phone_verification) {
|
||||
throw new HttpError(
|
||||
403,
|
||||
@@ -338,8 +323,11 @@ export const assertVerifiedAccount = (
|
||||
},
|
||||
);
|
||||
}
|
||||
if (emailLast) assertEmail();
|
||||
else assertCard();
|
||||
if (user?.requires_card_verification) {
|
||||
throw new HttpError(403, 'Please verify your card to continue', {
|
||||
legacyCode: 'card_verification_required',
|
||||
});
|
||||
}
|
||||
// A team seat signs in on a password its administrator still holds,
|
||||
// so it reaches nothing until it has replaced that password.
|
||||
if (user?.requires_password_change) {
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'node:events';
|
||||
import type { NextFunction, Request, RequestHandler, Response } from 'express';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { isHttpError } from '../HttpError';
|
||||
import {
|
||||
createLargeJsonBodyParser,
|
||||
DEFAULT_JSON_LIMIT_BYTES,
|
||||
} from './largeJsonBody';
|
||||
|
||||
const MiB = 1024 * 1024;
|
||||
|
||||
// A complete request with no socket reads as already consumed to body-parser,
|
||||
// so the parsers call straight through, leaving any preset `body` in place.
|
||||
const makeReq = (
|
||||
bytes: number | undefined,
|
||||
user: string | null = 'u1',
|
||||
body?: unknown,
|
||||
) =>
|
||||
({
|
||||
complete: true,
|
||||
body,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(bytes === undefined ? {} : { 'content-length': String(bytes) }),
|
||||
...(user ? { authorization: user } : {}),
|
||||
},
|
||||
}) as unknown as Request;
|
||||
|
||||
const makeRes = () => {
|
||||
const res = new EventEmitter() as EventEmitter & {
|
||||
setHeader: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
res.setHeader = vi.fn();
|
||||
return res;
|
||||
};
|
||||
|
||||
const authProbe = vi.fn<RequestHandler>((req, _res, next) => {
|
||||
if (req.headers.authorization) {
|
||||
(req as { actor?: unknown }).actor = {
|
||||
user: { uuid: req.headers.authorization },
|
||||
};
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
const run = (mw: RequestHandler, req: Request, res = makeRes()) =>
|
||||
new Promise<{ err: unknown; res: ReturnType<typeof makeRes> }>(
|
||||
(resolve) => {
|
||||
mw(
|
||||
req,
|
||||
res as unknown as Response,
|
||||
((err?: unknown) => resolve({ err, res })) as NextFunction,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const statusOf = (err: unknown) => (isHttpError(err) ? err.statusCode : err);
|
||||
|
||||
describe('createLargeJsonBodyParser', () => {
|
||||
const create = ({
|
||||
budgetBytes = 200 * MiB,
|
||||
perUserBytes = 128 * MiB,
|
||||
accepts,
|
||||
}: {
|
||||
budgetBytes?: number;
|
||||
perUserBytes?: number;
|
||||
accepts?: (req: Request) => boolean;
|
||||
} = {}) =>
|
||||
createLargeJsonBodyParser({
|
||||
authProbe,
|
||||
limitBytes: 128 * MiB,
|
||||
budgetBytes,
|
||||
perUserBytes,
|
||||
accepts,
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no Content-Length', undefined],
|
||||
['a body within the default limit', DEFAULT_JSON_LIMIT_BYTES],
|
||||
['a body over the large limit', 128 * MiB + 1],
|
||||
])('leaves %s to the global parser without probing', async (_, bytes) => {
|
||||
authProbe.mockClear();
|
||||
const { err } = await run(create(), makeReq(bytes));
|
||||
expect(err).toBeUndefined();
|
||||
expect(authProbe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves an unauthenticated large body to the global parser', async () => {
|
||||
const mw = create({ budgetBytes: 60 * MiB });
|
||||
const { err } = await run(mw, makeReq(60 * MiB, null));
|
||||
expect(err).toBeUndefined();
|
||||
// Nothing was charged, so a signed-in caller still fits.
|
||||
expect((await run(mw, makeReq(60 * MiB))).err).toBeUndefined();
|
||||
});
|
||||
|
||||
it('charges nothing for a client that left during the probe', async () => {
|
||||
const mw = create({ budgetBytes: 60 * MiB });
|
||||
const gone = Object.assign(makeRes(), { closed: true });
|
||||
expect((await run(mw, makeReq(60 * MiB), gone)).err).toBeUndefined();
|
||||
expect((await run(mw, makeReq(60 * MiB))).err).toBeUndefined();
|
||||
});
|
||||
|
||||
it('refuses with 503 past the budget until a response closes', async () => {
|
||||
const mw = create({ budgetBytes: 130 * MiB });
|
||||
const first = await run(mw, makeReq(60 * MiB, 'u1'));
|
||||
await run(mw, makeReq(60 * MiB, 'u2'));
|
||||
|
||||
const refused = await run(mw, makeReq(60 * MiB, 'u3'));
|
||||
expect(statusOf(refused.err)).toBe(503);
|
||||
// Load shedding is recorded, not paged.
|
||||
expect(isHttpError(refused.err) && refused.err.alarmSeverity).toBe(
|
||||
'info',
|
||||
);
|
||||
expect(refused.res.setHeader).toHaveBeenCalledWith('Retry-After', '5');
|
||||
|
||||
first.res.emit('close');
|
||||
expect((await run(mw, makeReq(60 * MiB, 'u3'))).err).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses one user's bodies with 429 past their share", async () => {
|
||||
const mw = create({ perUserBytes: 100 * MiB });
|
||||
const first = await run(mw, makeReq(60 * MiB, 'u1'));
|
||||
|
||||
const refused = await run(mw, makeReq(60 * MiB, 'u1'));
|
||||
expect(statusOf(refused.err)).toBe(429);
|
||||
expect(refused.res.setHeader).toHaveBeenCalledWith('Retry-After', '5');
|
||||
// Another user isn't held back by it.
|
||||
expect((await run(mw, makeReq(60 * MiB, 'u2'))).err).toBeUndefined();
|
||||
|
||||
first.res.emit('close');
|
||||
expect((await run(mw, makeReq(60 * MiB, 'u1'))).err).toBeUndefined();
|
||||
});
|
||||
|
||||
it('refuses with 413 a parsed body `accepts` turns down', async () => {
|
||||
const mw = create({
|
||||
accepts: (req) => (req.body as { ok?: boolean }).ok === true,
|
||||
});
|
||||
const refused = await run(mw, makeReq(60 * MiB, 'u1', { ok: false }));
|
||||
expect(statusOf(refused.err)).toBe(413);
|
||||
expect(
|
||||
(await run(mw, makeReq(60 * MiB, 'u2', { ok: true }))).err,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,134 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import express, { type Request, type RequestHandler } from 'express';
|
||||
import { actorUid } from '../../actor';
|
||||
import { HttpError } from '../HttpError';
|
||||
|
||||
/** The global JSON parser's limit, which applies before auth. */
|
||||
export const DEFAULT_JSON_LIMIT_BYTES = 50 * 1024 * 1024;
|
||||
|
||||
interface LargeJsonBodyOptions {
|
||||
/** The global auth probe; it skips requests that already carry an actor. */
|
||||
authProbe: RequestHandler;
|
||||
limitBytes: number;
|
||||
/** Declared bytes of large bodies a process holds at once. */
|
||||
budgetBytes: number;
|
||||
/** Declared bytes of large bodies one user holds at once. */
|
||||
perUserBytes: number;
|
||||
/** Checked once the body is parsed; a body it refuses gets a 413. */
|
||||
accepts?: (req: Request) => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses JSON bodies over the default limit, up to `limitBytes`, once the
|
||||
* caller's header or cookie credential resolves. Unauthenticated requests and
|
||||
* requests without a Content-Length fall through to the global parser and its
|
||||
* limit.
|
||||
*
|
||||
* The parsed body lives as long as the request, so a body's bytes count against
|
||||
* both budgets until its response closes. Over the user's budget gets a 429,
|
||||
* over the process budget a 503.
|
||||
*/
|
||||
export const createLargeJsonBodyParser = ({
|
||||
authProbe,
|
||||
limitBytes,
|
||||
budgetBytes,
|
||||
perUserBytes,
|
||||
accepts,
|
||||
}: LargeJsonBodyOptions): RequestHandler => {
|
||||
const json = express.json({ limit: limitBytes });
|
||||
const textAsJson = express.json({
|
||||
limit: limitBytes,
|
||||
type: (req) =>
|
||||
req.headers['content-type'] === 'text/plain;actually=json',
|
||||
});
|
||||
let inFlightBytes = 0;
|
||||
const inFlightByUser = new Map<string, number>();
|
||||
|
||||
return (req, res, next) => {
|
||||
const declared = Number(req.headers['content-length']);
|
||||
if (!(declared > DEFAULT_JSON_LIMIT_BYTES && declared <= limitBytes)) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
void authProbe(req, res, () => {
|
||||
// `res.closed`: the client left during the probe, so `close` has
|
||||
// already fired and would never release a charge.
|
||||
if (!req.actor || res.closed) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
const user = req.actor.user.uuid ?? actorUid(req.actor);
|
||||
const userBytes = inFlightByUser.get(user) ?? 0;
|
||||
if (userBytes + declared > perUserBytes) {
|
||||
res.setHeader('Retry-After', '5');
|
||||
next(
|
||||
new HttpError(
|
||||
429,
|
||||
'Too many large requests in progress for this account',
|
||||
{ legacyCode: 'too_many_requests' },
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (inFlightBytes + declared > budgetBytes) {
|
||||
res.setHeader('Retry-After', '5');
|
||||
next(
|
||||
new HttpError(503, 'Too many large requests in progress', {
|
||||
code: 'large_body_capacity',
|
||||
// Load shedding, not a fault: record it, don't page.
|
||||
alarmSeverity: 'info',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
inFlightBytes += declared;
|
||||
inFlightByUser.set(user, userBytes + declared);
|
||||
res.once('close', () => {
|
||||
inFlightBytes -= declared;
|
||||
const left = (inFlightByUser.get(user) ?? 0) - declared;
|
||||
if (left > 0) inFlightByUser.set(user, left);
|
||||
else inFlightByUser.delete(user);
|
||||
});
|
||||
json(req, res, (err?: unknown) => {
|
||||
if (err) {
|
||||
next(err);
|
||||
return;
|
||||
}
|
||||
textAsJson(req, res, (err?: unknown) => {
|
||||
if (err) {
|
||||
next(err);
|
||||
return;
|
||||
}
|
||||
if (accepts && !accepts(req)) {
|
||||
// The same answer the default limit gives.
|
||||
next(
|
||||
new HttpError(413, 'request entity too large', {
|
||||
legacyCode: 'client_bad_request',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
next();
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
};
|
||||
@@ -476,8 +476,9 @@ function actorMatchesExpectedApp(
|
||||
}
|
||||
|
||||
/**
|
||||
* Fresh authenticated identities replace the sticky public billing cookie when
|
||||
* the visitor switches accounts.
|
||||
* Mirror of `resolvePrivateIdentity` for public hosted apps. Reads the sticky
|
||||
* `puter.public.hosted.actor.token` cookie first, then the same
|
||||
* session/bootstrap fallbacks.
|
||||
*/
|
||||
export async function resolvePublicHostedIdentity(opts: {
|
||||
req: Request;
|
||||
@@ -499,6 +500,37 @@ export async function resolvePublicHostedIdentity(opts: {
|
||||
const cookies = (req as Request & { cookies?: Record<string, string> })
|
||||
.cookies;
|
||||
|
||||
const { name: publicCookieNameV2, legacyName: publicCookieNameLegacy } =
|
||||
HOSTED_ASSET_COOKIES.public;
|
||||
const publicCookieToken =
|
||||
(typeof cookies?.[publicCookieNameV2] === 'string'
|
||||
? cookies[publicCookieNameV2]
|
||||
: null) ??
|
||||
(typeof cookies?.[publicCookieNameLegacy] === 'string'
|
||||
? cookies[publicCookieNameLegacy]
|
||||
: null);
|
||||
if (publicCookieToken) {
|
||||
try {
|
||||
const claims = await authService.verifyHostedAssetToken(
|
||||
'public',
|
||||
publicCookieToken,
|
||||
{
|
||||
appUid: expectedAppUid,
|
||||
subdomain: expectedSubdomain,
|
||||
host: expectedHost,
|
||||
},
|
||||
);
|
||||
return {
|
||||
source: 'public-cookie',
|
||||
userUid: claims.userUid,
|
||||
sessionUuid: claims.sessionUuid,
|
||||
hasValidPublicCookie: true,
|
||||
};
|
||||
} catch {
|
||||
/* fall through — v1-signed cookies land here and get re-minted */
|
||||
}
|
||||
}
|
||||
|
||||
const existingActor = req.actor;
|
||||
if (
|
||||
existingActor?.user?.uuid &&
|
||||
@@ -554,37 +586,6 @@ export async function resolvePublicHostedIdentity(opts: {
|
||||
}
|
||||
}
|
||||
|
||||
const { name: publicCookieNameV2, legacyName: publicCookieNameLegacy } =
|
||||
HOSTED_ASSET_COOKIES.public;
|
||||
const publicCookieToken =
|
||||
(typeof cookies?.[publicCookieNameV2] === 'string'
|
||||
? cookies[publicCookieNameV2]
|
||||
: null) ??
|
||||
(typeof cookies?.[publicCookieNameLegacy] === 'string'
|
||||
? cookies[publicCookieNameLegacy]
|
||||
: null);
|
||||
if (publicCookieToken) {
|
||||
try {
|
||||
const claims = await authService.verifyHostedAssetToken(
|
||||
'public',
|
||||
publicCookieToken,
|
||||
{
|
||||
appUid: expectedAppUid,
|
||||
subdomain: expectedSubdomain,
|
||||
host: expectedHost,
|
||||
},
|
||||
);
|
||||
return {
|
||||
source: 'public-cookie',
|
||||
userUid: claims.userUid,
|
||||
sessionUuid: claims.sessionUuid,
|
||||
hasValidPublicCookie: true,
|
||||
};
|
||||
} catch {
|
||||
/* fall through — v1-signed cookies land here and get re-minted */
|
||||
}
|
||||
}
|
||||
|
||||
return { source: 'none' };
|
||||
}
|
||||
|
||||
|
||||
@@ -2591,239 +2591,3 @@ describe('createPuterSiteMiddleware — private-app gate decisions', () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('public hosted asset billing identity', () => {
|
||||
it.each(['actor', 'session', 'query'] as const)(
|
||||
'replaces an older billing cookie with the current %s identity',
|
||||
async (source) => {
|
||||
const owner = await makeUserWithHome();
|
||||
const previous = await makeUser();
|
||||
const current = await makeUser();
|
||||
const home = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}`,
|
||||
))!;
|
||||
const sub = `switch-${uuidv4().slice(0, 8)}`;
|
||||
const host = `${sub}.site.puter.localhost`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: home.id,
|
||||
});
|
||||
await writeFile(
|
||||
owner.id,
|
||||
`/${owner.username}/index.html`,
|
||||
Buffer.from('hello'),
|
||||
);
|
||||
const app = await server.stores.app.create(
|
||||
{ name: sub, title: sub, index_url: `http://${host}/` },
|
||||
{ ownerUserId: owner.id },
|
||||
);
|
||||
const previousToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(previous),
|
||||
app.uid,
|
||||
);
|
||||
const previousActor =
|
||||
(await server.services.auth.authenticateFromToken(
|
||||
previousToken,
|
||||
))!;
|
||||
const currentToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(current),
|
||||
app.uid,
|
||||
);
|
||||
const oldCookie = await server.services.auth.createHostedAssetToken(
|
||||
'public',
|
||||
{
|
||||
appUid: app.uid,
|
||||
userUid: previous.uuid,
|
||||
sessionUuid: previousActor.session?.uid,
|
||||
subdomain: sub,
|
||||
host,
|
||||
},
|
||||
);
|
||||
const cookieName = HOSTED_ASSET_COOKIES.public.name;
|
||||
const req = makeReq({
|
||||
hostname: host,
|
||||
path: '/index.html',
|
||||
cookies: {
|
||||
[cookieName]: oldCookie,
|
||||
...(source === 'session' ? { session: currentToken } : {}),
|
||||
},
|
||||
query:
|
||||
source === 'query'
|
||||
? { 'puter.auth.token': currentToken }
|
||||
: {},
|
||||
});
|
||||
if (source === 'actor')
|
||||
req.actor =
|
||||
(await server.services.auth.authenticateFromToken(
|
||||
currentToken,
|
||||
))!;
|
||||
const { res, out } = makeRes();
|
||||
await buildMiddleware({ cookie_name: 'session' })(
|
||||
req,
|
||||
res,
|
||||
vi.fn(),
|
||||
);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(out.statusCode).toBe(200);
|
||||
expect(req.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
const refreshed = out.cookies[cookieName];
|
||||
expect(refreshed).toBeTruthy();
|
||||
expect(
|
||||
(
|
||||
await server.services.auth.verifyHostedAssetToken(
|
||||
'public',
|
||||
refreshed,
|
||||
{ appUid: app.uid, host },
|
||||
)
|
||||
).userUid,
|
||||
).toBe(current.uuid);
|
||||
const nextReq = makeReq({
|
||||
hostname: host,
|
||||
path: '/index.html',
|
||||
cookies: { [cookieName]: refreshed },
|
||||
});
|
||||
await runMiddleware(buildMiddleware(), nextReq);
|
||||
expect(nextReq.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
},
|
||||
);
|
||||
|
||||
it('bills recognized visitors, reuses the parent session, and falls back to the owner', async () => {
|
||||
const owner = await makeUserWithHome();
|
||||
const visitor = await makeUser();
|
||||
const home = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}`,
|
||||
))!;
|
||||
const sub = `egress-${uuidv4().slice(0, 8)}`;
|
||||
const host = `${sub}.site.puter.localhost`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: home.id,
|
||||
});
|
||||
await writeFile(
|
||||
owner.id,
|
||||
`/${owner.username}/index.html`,
|
||||
Buffer.from('hello'),
|
||||
);
|
||||
const app = await server.stores.app.create(
|
||||
{ name: sub, title: sub, index_url: `http://${host}/` },
|
||||
{ ownerUserId: owner.id },
|
||||
);
|
||||
const token = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
app.uid,
|
||||
);
|
||||
const authenticated =
|
||||
await server.services.auth.authenticateFromToken(token);
|
||||
const serve = async (query = {}, cookies = {}, headers = {}) => {
|
||||
const req = makeReq({ hostname: host, path: '/index.html', query });
|
||||
Object.assign(req, { cookies, headers });
|
||||
const { res, out } = makeRes();
|
||||
await buildMiddleware()(req, res, vi.fn());
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(out.statusCode).toBe(200);
|
||||
return { req, out };
|
||||
};
|
||||
const create = vi.spyOn(server.stores.session, 'create');
|
||||
try {
|
||||
const first = await serve({ 'puter.auth.token': token });
|
||||
expect(first.req.egressActor?.user?.uuid).toBe(visitor.uuid);
|
||||
const cookieName = HOSTED_ASSET_COOKIES.public.name;
|
||||
const cookie = first.out.cookies[cookieName];
|
||||
expect(cookie).toBeTruthy();
|
||||
const claims = await server.services.auth.verifyHostedAssetToken(
|
||||
'public',
|
||||
cookie,
|
||||
{ appUid: app.uid, host },
|
||||
);
|
||||
expect(claims.sessionUuid).toBe(authenticated?.session?.uid);
|
||||
const next = await serve({}, { [cookieName]: cookie });
|
||||
expect(next.req.egressActor?.user?.uuid).toBe(visitor.uuid);
|
||||
expect(next.out.cookies).toEqual({});
|
||||
const blockedCookies = await serve(
|
||||
{},
|
||||
{},
|
||||
{ referer: `http://${host}/?puter.auth.token=${token}` },
|
||||
);
|
||||
expect(blockedCookies.req.egressActor?.user?.uuid).toBe(
|
||||
visitor.uuid,
|
||||
);
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
const anonymous = await serve();
|
||||
expect(anonymous.req.egressActor?.user?.uuid).toBe(owner.uuid);
|
||||
const invalid = await serve({}, { [cookieName]: 'invalid' });
|
||||
expect(invalid.req.egressActor?.user?.uuid).toBe(owner.uuid);
|
||||
await server.services.auth.revokeAllSessionsForUserId(visitor.id);
|
||||
const revoked = await serve({}, { [cookieName]: cookie });
|
||||
expect(revoked.req.egressActor?.user?.uuid).toBe(owner.uuid);
|
||||
} finally {
|
||||
create.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('site rooted in the owner tree, root moved to Trash', () => {
|
||||
it('stops serving a trashed folder and resumes when restored', async () => {
|
||||
const owner = await makeUserWithHome();
|
||||
const dir = await server.services.fs.mkdir(owner.id, {
|
||||
path: `/${owner.username}/Documents/site`,
|
||||
createMissingParents: true,
|
||||
});
|
||||
const body = Buffer.from('<html>trashed</html>');
|
||||
await server.services.fs.write(owner.id, {
|
||||
fileMetadata: {
|
||||
path: `/${owner.username}/Documents/site/index.html`,
|
||||
size: body.byteLength,
|
||||
},
|
||||
fileContent: body,
|
||||
});
|
||||
const sub = `tr-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: dir.id,
|
||||
});
|
||||
const serveIndex = async () => {
|
||||
const mw = createPuterSiteMiddleware(hostingConfig, {
|
||||
clients: server.clients,
|
||||
stores: server.stores,
|
||||
services: server.services,
|
||||
});
|
||||
const { res, out } = makeRes();
|
||||
await mw(
|
||||
makeReq({
|
||||
hostname: `${sub}.site.puter.localhost`,
|
||||
path: '/',
|
||||
}),
|
||||
res,
|
||||
vi.fn(),
|
||||
);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
return out;
|
||||
};
|
||||
expect((await serveIndex()).statusCode).toBe(200);
|
||||
|
||||
const trash = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}/Trash`,
|
||||
))!;
|
||||
const entry = (await server.stores.fsEntry.getEntryById(dir.id))!;
|
||||
await server.services.fs.move(owner.id, {
|
||||
source: entry,
|
||||
destinationParent: trash,
|
||||
});
|
||||
const moved = (await server.stores.fsEntry.getEntryById(dir.id))!;
|
||||
expect(moved.path.startsWith(`/${owner.username}/Trash/`)).toBe(true);
|
||||
|
||||
const out = await serveIndex();
|
||||
expect(out.statusCode).toBe(404);
|
||||
const documents = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}/Documents`,
|
||||
))!;
|
||||
await server.services.fs.move(owner.id, {
|
||||
source: moved,
|
||||
destinationParent: documents,
|
||||
});
|
||||
expect((await serveIndex()).statusCode).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -27,7 +27,6 @@ import { MANAGE_PERM_PREFIX } from '../../../services/permission/consts';
|
||||
import type { puterStores } from '../../../stores';
|
||||
import type { IConfig, LayerInstances } from '../../../types';
|
||||
import { awaitLater } from '../../../util/concurrency';
|
||||
import { isTrashedPath } from '../../../util/hostedAppBacking';
|
||||
import { makeActor } from '../../actor';
|
||||
import {
|
||||
buildAppCenterFallback,
|
||||
@@ -110,6 +109,10 @@ const isServableEntryPath = (
|
||||
return !isWorkersSourcePath(relPath) && !isSiteConfigPath(relPath);
|
||||
};
|
||||
|
||||
/** Inside some owner's top-level Trash, which is where Delete puts things. */
|
||||
const isTrashedPath = (path: string): boolean =>
|
||||
/^\/[^/]+\/Trash(\/|$)/u.test(path);
|
||||
|
||||
/**
|
||||
* Suggested value for `config.hosting_csp`. Not applied unless configured.
|
||||
*
|
||||
@@ -330,7 +333,6 @@ export const createPuterSiteMiddleware = (
|
||||
|
||||
const associatedApp = (await appLookup) as AppRow | null;
|
||||
|
||||
let hostedVisitorUid: string | undefined;
|
||||
const isPrivateApp = Boolean(associatedApp?.is_private);
|
||||
const privateApp = isPrivateApp ? associatedApp : null;
|
||||
|
||||
@@ -398,8 +400,6 @@ export const createPuterSiteMiddleware = (
|
||||
return;
|
||||
}
|
||||
|
||||
hostedVisitorUid = identity.userUid;
|
||||
|
||||
// Entitlement check runs on every request — matching v1. The
|
||||
// sticky cookie is an identity shortcut, not an access cache;
|
||||
// the marketplace extension already caches access decisions
|
||||
@@ -509,8 +509,11 @@ export const createPuterSiteMiddleware = (
|
||||
res.status(404).type('text/plain').send('Subdomain not found');
|
||||
return;
|
||||
} else {
|
||||
// Remember the authenticated visitor for asset billing across
|
||||
// requests that do not carry the original app token.
|
||||
// Public hosted site. Mint the public hosted-actor cookie if
|
||||
// we can identify the visitor — lets the hosted page make
|
||||
// cross-origin requests as the actor without needing a
|
||||
// host-scoped main session cookie. No-op for anonymous
|
||||
// visitors.
|
||||
try {
|
||||
const identity = await resolvePublicHostedIdentity({
|
||||
req,
|
||||
@@ -523,7 +526,6 @@ export const createPuterSiteMiddleware = (
|
||||
expectedSubdomain: subdomain,
|
||||
expectedHost: host,
|
||||
});
|
||||
hostedVisitorUid = identity.userUid;
|
||||
if (
|
||||
identity.userUid &&
|
||||
!(identity as { hasValidPublicCookie?: boolean })
|
||||
@@ -572,7 +574,7 @@ export const createPuterSiteMiddleware = (
|
||||
.send(SUBDOMAIN_404);
|
||||
return;
|
||||
}
|
||||
if (!rootEntry.isDir || isTrashedPath(rootEntry.path)) {
|
||||
if (!rootEntry.isDir) {
|
||||
res.status(404)
|
||||
.type('text/html; charset=UTF-8')
|
||||
.send(SUBDOMAIN_404);
|
||||
@@ -582,27 +584,37 @@ export const createPuterSiteMiddleware = (
|
||||
// A site rooted in someone else's directory stands on the `manage`
|
||||
// grant that authorized publishing it, and serves that whole subtree
|
||||
// with the ACL bypassed — so the grant is re-read on every request.
|
||||
// Trash is its own check: a grant is keyed on the node, not its
|
||||
// location, so it survives the owner deleting the directory. Sites
|
||||
// rooted in their publisher's own tree skip all of this.
|
||||
if (rootEntry.userId !== site.user_id) {
|
||||
let stillPublishable = false;
|
||||
try {
|
||||
stillPublishable = await layers.services.acl.check(
|
||||
makeActor({
|
||||
user: {
|
||||
id: owner.id,
|
||||
uuid: owner.uuid,
|
||||
username: owner.username,
|
||||
if (!isTrashedPath(rootEntry.path)) {
|
||||
try {
|
||||
stillPublishable = await layers.services.acl.check(
|
||||
makeActor({
|
||||
user: {
|
||||
id: owner.id,
|
||||
uuid: owner.uuid,
|
||||
username: owner.username,
|
||||
},
|
||||
}),
|
||||
{
|
||||
path: rootEntry.path,
|
||||
resolveAncestors: () =>
|
||||
layers.services.fs.getAncestorChain(
|
||||
rootEntry.path,
|
||||
),
|
||||
},
|
||||
}),
|
||||
{
|
||||
path: rootEntry.path,
|
||||
resolveAncestors: () =>
|
||||
layers.services.fs.getAncestorChain(rootEntry.path),
|
||||
},
|
||||
MANAGE_PERM_PREFIX,
|
||||
);
|
||||
} catch (e) {
|
||||
// Fail closed — an unreadable grant is not a held one.
|
||||
console.warn('[puter-site] publish grant recheck failed', e);
|
||||
MANAGE_PERM_PREFIX,
|
||||
);
|
||||
} catch (e) {
|
||||
// Fail closed — an unreadable grant is not a held one.
|
||||
console.warn(
|
||||
'[puter-site] publish grant recheck failed',
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!stillPublishable) {
|
||||
res.status(404)
|
||||
@@ -825,22 +837,7 @@ export const createPuterSiteMiddleware = (
|
||||
// hosting is unauthenticated by design, so most visitors are nobody in
|
||||
// particular. Set unconditionally because hosting subdomains are not
|
||||
// metered by default: this is what opts the response in.
|
||||
let visitorActor = req.actor;
|
||||
if (hostedVisitorUid && visitorActor?.user?.uuid !== hostedVisitorUid) {
|
||||
try {
|
||||
const visitor =
|
||||
await layers.stores.user.getByUuid(hostedVisitorUid);
|
||||
if (visitor && !visitor.suspended) {
|
||||
visitorActor = makeActor({
|
||||
user: visitor,
|
||||
app: associatedApp ?? undefined,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Unknown requesters are billed to the site owner.
|
||||
}
|
||||
}
|
||||
req.egressActor = visitorActor ?? {
|
||||
req.egressActor = req.actor ?? {
|
||||
user: {
|
||||
uuid: owner.uuid,
|
||||
id: owner.id,
|
||||
|
||||
@@ -84,10 +84,6 @@ export function sweepMemoryWindows() {
|
||||
)
|
||||
memoryWindows.delete(k);
|
||||
}
|
||||
for (const [k, entry] of memoryCounters) {
|
||||
if (entry.index < Math.floor(now / entry.windowMs) - 1)
|
||||
memoryCounters.delete(k);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
@@ -159,97 +155,6 @@ async function peekMemory(key, limit, windowMs) {
|
||||
return timestamps.length < limit;
|
||||
}
|
||||
|
||||
// -- Counted claims --------------------------------------------------
|
||||
//
|
||||
// `claim` keeps a member per unit, which is fine for a few units and not for
|
||||
// thousands. The counted form keeps two counters per key, the current fixed
|
||||
// window and the one before it, and slides by weighting the previous one by
|
||||
// how much of it the window still covers. It can be off by a little at window
|
||||
// edges, and its buckets are separate from the other forms'.
|
||||
|
||||
/** The sliding-window estimate from the two fixed windows it overlaps. */
|
||||
const slidingCount = (previous, current, now, windowMs) =>
|
||||
previous * (1 - (now % windowMs) / windowMs) + current;
|
||||
|
||||
/** How many of `units` a counted claim that left `used` spent may keep. */
|
||||
const countedAdmits = (used, limit, units) =>
|
||||
units - Math.min(units, Math.max(0, Math.ceil(used - limit)));
|
||||
|
||||
/** Key → `{ index, current, previous, windowMs }`, swept with the windows. */
|
||||
const memoryCounters = new Map();
|
||||
|
||||
async function claimCountMemory(key, limit, windowMs, units) {
|
||||
const now = Date.now();
|
||||
const index = Math.floor(now / windowMs);
|
||||
let entry = memoryCounters.get(key);
|
||||
if (!entry) {
|
||||
if (memoryCounters.size >= MEMORY_MAX_KEYS) {
|
||||
memoryCounters.delete(memoryCounters.keys().next().value);
|
||||
}
|
||||
entry = { index, current: 0, previous: 0, windowMs };
|
||||
memoryCounters.set(key, entry);
|
||||
} else if (entry.index !== index) {
|
||||
entry.previous = entry.index === index - 1 ? entry.current : 0;
|
||||
entry.current = 0;
|
||||
entry.index = index;
|
||||
}
|
||||
entry.windowMs = windowMs;
|
||||
const used = slidingCount(entry.previous, entry.current, now, windowMs);
|
||||
const admitted = countedAdmits(used + units, limit, units);
|
||||
entry.current += admitted;
|
||||
return admitted;
|
||||
}
|
||||
|
||||
async function claimCountRedis(redis, key, limit, windowMs, units) {
|
||||
const now = Date.now();
|
||||
const index = Math.floor(now / windowMs);
|
||||
// The hash tag keeps both windows in one slot, so one MULTI reads both.
|
||||
const current = `ratec:{${key}}:${index}`;
|
||||
const results = await redis
|
||||
.multi()
|
||||
.incrby(current, units)
|
||||
.pexpire(current, 2 * windowMs)
|
||||
.get(`ratec:{${key}}:${index - 1}`)
|
||||
.exec();
|
||||
const used = slidingCount(
|
||||
Number(multiResult(results, 2) ?? 0),
|
||||
Number(multiResult(results, 0)),
|
||||
now,
|
||||
windowMs,
|
||||
);
|
||||
const admitted = countedAdmits(used, limit, units);
|
||||
if (admitted < units) await redis.decrby(current, units - admitted);
|
||||
return admitted;
|
||||
}
|
||||
|
||||
async function claimCountKv(kv, key, limit, windowMs, units) {
|
||||
const now = Date.now();
|
||||
const index = Math.floor(now / windowMs);
|
||||
const current = `ratec:${key}:${index}`;
|
||||
const [counted, previous] = await Promise.all([
|
||||
kv.incr({
|
||||
key: current,
|
||||
pathAndAmountMap: { '': units },
|
||||
expireAt: Math.ceil((now + 2 * windowMs) / 1000),
|
||||
}),
|
||||
kv.get({ key: `ratec:${key}:${index - 1}` }),
|
||||
]);
|
||||
const used = slidingCount(
|
||||
Number(previous.res ?? 0),
|
||||
Number(counted.res),
|
||||
now,
|
||||
windowMs,
|
||||
);
|
||||
const admitted = countedAdmits(used, limit, units);
|
||||
if (admitted < units) {
|
||||
await kv.decr({
|
||||
key: current,
|
||||
pathAndAmountMap: { '': units - admitted },
|
||||
});
|
||||
}
|
||||
return admitted;
|
||||
}
|
||||
|
||||
// -- Redis backend ---------------------------------------------------
|
||||
|
||||
// ioredis MULTI/EXEC reports per-command failures inside the exec() result
|
||||
@@ -570,10 +475,6 @@ function instrumentBackendPair(name, pair) {
|
||||
withSpan('rate_limit.claim', attrs, () =>
|
||||
pair.claim(key, limit, windowMs, units),
|
||||
),
|
||||
count: (key, limit, windowMs, units) =>
|
||||
withSpan('rate_limit.count', attrs, () =>
|
||||
pair.count(key, limit, windowMs, units),
|
||||
),
|
||||
acquire: (key, limit) =>
|
||||
withSpan('rate_limit.acquire', attrs, () =>
|
||||
pair.acquire(key, limit),
|
||||
@@ -585,7 +486,6 @@ const memoryBackendPair = instrumentBackendPair('memory', {
|
||||
rate: checkMemory,
|
||||
peek: peekMemory,
|
||||
claim: claimMemory,
|
||||
count: claimCountMemory,
|
||||
acquire: acquireMemoryConcurrent,
|
||||
});
|
||||
|
||||
@@ -636,8 +536,6 @@ export function configureRateLimit({
|
||||
peekRedis(redis, key, limit, windowMs),
|
||||
claim: (key, limit, windowMs, units) =>
|
||||
claimRedis(redis, key, limit, windowMs, units),
|
||||
count: (key, limit, windowMs, units) =>
|
||||
claimCountRedis(redis, key, limit, windowMs, units),
|
||||
acquire: (key, limit) => acquireRedisConcurrent(redis, key, limit),
|
||||
});
|
||||
}
|
||||
@@ -647,8 +545,6 @@ export function configureRateLimit({
|
||||
peek: (key, limit) => peekKv(kv, key, limit),
|
||||
claim: (key, limit, windowMs, units) =>
|
||||
claimKv(kv, key, limit, windowMs, units),
|
||||
count: (key, limit, windowMs, units) =>
|
||||
claimCountKv(kv, key, limit, windowMs, units),
|
||||
acquire: (key, limit) => acquireKvConcurrent(kv, key, limit),
|
||||
});
|
||||
}
|
||||
@@ -978,32 +874,6 @@ export async function claimRateLimit(key, limit, windowMs, units, backend) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `claimRateLimit` for budgets measured in many units per call: a claim costs
|
||||
* the same however many units it spends (see "Counted claims" above). Its
|
||||
* buckets are its own, so a key is only ever claimed one way. Fails open,
|
||||
* admitting every unit.
|
||||
*/
|
||||
export async function claimCountedRateLimit(
|
||||
key,
|
||||
limit,
|
||||
windowMs,
|
||||
units,
|
||||
backend,
|
||||
) {
|
||||
if (!(units > 0)) return 0;
|
||||
const bk = resolveBackend(backend);
|
||||
try {
|
||||
return await bk.count(key, limit, windowMs, units);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
'[rate-limit] imperative counted claim failed, failing open:',
|
||||
err,
|
||||
);
|
||||
return units;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Charge a route-shaped rate-limit spec from inside a handler.
|
||||
*
|
||||
@@ -1179,8 +1049,8 @@ export function concurrencyGate(opts) {
|
||||
* Acquire a concurrent slot for a driver call. Mirrors `checkDriverRateLimit`
|
||||
* but returns an acquisition handle: caller MUST invoke `release()` on the
|
||||
* returned object exactly once, even on thrown errors — typically in a
|
||||
* `finally`. `renew()` keeps a long-held slot past the orphan TTL. `ok: false`
|
||||
* means the slot was full; callers should reject with 429 in that case.
|
||||
* `finally`. `ok: false` means the slot was full; callers should reject with
|
||||
* 429 in that case.
|
||||
*
|
||||
* `opts` is the resolved per-method spec from the driver's decorator (or
|
||||
* imperative `concurrent` field). Omitting `opts` (driver declares nothing)
|
||||
@@ -1190,7 +1060,7 @@ export function concurrencyGate(opts) {
|
||||
*/
|
||||
export async function acquireDriverConcurrent(req, ifaceName, method, opts) {
|
||||
if (!opts || typeof opts.limit !== 'number') {
|
||||
return { ok: true, release: noop, renew: noop };
|
||||
return { ok: true, release: noop };
|
||||
}
|
||||
const slot = await charge(
|
||||
req,
|
||||
@@ -1202,7 +1072,8 @@ export async function acquireDriverConcurrent(req, ifaceName, method, opts) {
|
||||
label: '[concurrent] driver acquire',
|
||||
},
|
||||
);
|
||||
if (!slot) return { ok: true, release: noop, renew: noop };
|
||||
if (!slot.ok) return { ok: false, release: noop, renew: noop };
|
||||
return guardSlot(slot, '[concurrent] driver');
|
||||
if (!slot) return { ok: true, release: noop };
|
||||
if (!slot.ok) return { ok: false, release: noop };
|
||||
const { release } = guardSlot(slot, '[concurrent] driver');
|
||||
return { ok: true, release };
|
||||
}
|
||||
@@ -37,7 +37,6 @@ import {
|
||||
acquireDriverConcurrent,
|
||||
checkDriverRateLimit,
|
||||
checkRateLimit,
|
||||
claimCountedRateLimit,
|
||||
claimRateLimit,
|
||||
concurrencyGate,
|
||||
consumeRouteRateLimit,
|
||||
@@ -1488,13 +1487,6 @@ describe('rate + concurrent limiting — kv backend', () => {
|
||||
expect(await claimRateLimit(key, 5, 60_000, 1)).toBe(0);
|
||||
});
|
||||
|
||||
it('claims counted units, admitting only what fits', async () => {
|
||||
const key = uniqueScope();
|
||||
expect(await claimCountedRateLimit(key, 5, 60_000, 3)).toBe(3);
|
||||
expect(await claimCountedRateLimit(key, 5, 60_000, 3)).toBe(2);
|
||||
expect(await claimCountedRateLimit(key, 5, 60_000, 1)).toBe(0);
|
||||
});
|
||||
|
||||
it('claims against a full bucket as full while expired rows are unswept', async () => {
|
||||
const key = uniqueScope();
|
||||
await seedUnswept(`rate:${key}:`, 3);
|
||||
@@ -2018,103 +2010,6 @@ describe('acquireConcurrent (imperative)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('claimCountedRateLimit', () => {
|
||||
const uniqueKey = () =>
|
||||
`counted-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const WINDOW = 10_000;
|
||||
/** The start of a fixed window, so nothing before it carries in. */
|
||||
const windowStart = (n) => vi.setSystemTime(n * WINDOW);
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe.each([
|
||||
['memory', () => configureRateLimit()],
|
||||
[
|
||||
'redis',
|
||||
() =>
|
||||
configureRateLimit({
|
||||
default: 'redis',
|
||||
redis: new RedisMock(),
|
||||
}),
|
||||
],
|
||||
])('%s backend', (_name, configure) => {
|
||||
beforeEach(() => {
|
||||
configure();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
windowStart(1_000);
|
||||
});
|
||||
|
||||
it('admits up to the limit and spends only what fits', async () => {
|
||||
const key = uniqueKey();
|
||||
expect(await claimCountedRateLimit(key, 5, WINDOW, 3)).toBe(3);
|
||||
expect(await claimCountedRateLimit(key, 5, WINDOW, 3)).toBe(2);
|
||||
expect(await claimCountedRateLimit(key, 5, WINDOW, 1)).toBe(0);
|
||||
});
|
||||
|
||||
it('spends thousands of units in one claim', async () => {
|
||||
const key = uniqueKey();
|
||||
expect(
|
||||
await claimCountedRateLimit(key, 15_000, WINDOW, 6_000),
|
||||
).toBe(6_000);
|
||||
expect(
|
||||
await claimCountedRateLimit(key, 15_000, WINDOW, 10_000),
|
||||
).toBe(9_000);
|
||||
});
|
||||
|
||||
it('carries the previous window by how much of it is still inside', async () => {
|
||||
const key = uniqueKey();
|
||||
expect(await claimCountedRateLimit(key, 100, WINDOW, 100)).toBe(
|
||||
100,
|
||||
);
|
||||
// A quarter into the next window, 75 of those still count.
|
||||
vi.setSystemTime(1_001 * WINDOW + WINDOW / 4);
|
||||
expect(await claimCountedRateLimit(key, 100, WINDOW, 50)).toBe(25);
|
||||
// Two windows on, nothing carries.
|
||||
windowStart(1_003);
|
||||
expect(await claimCountedRateLimit(key, 100, WINDOW, 100)).toBe(
|
||||
100,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps its buckets apart from claimRateLimit', async () => {
|
||||
const key = uniqueKey();
|
||||
expect(await claimCountedRateLimit(key, 2, WINDOW, 2)).toBe(2);
|
||||
expect(await claimRateLimit(key, 2, WINDOW, 2)).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
it('claims nothing for zero units', async () => {
|
||||
const key = uniqueKey();
|
||||
expect(await claimCountedRateLimit(key, 1, WINDOW, 0)).toBe(0);
|
||||
expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
|
||||
});
|
||||
|
||||
it('fails open, admitting every unit, when the backend throws', async () => {
|
||||
const redis = new RedisMock();
|
||||
redis.multi = () => {
|
||||
throw new Error('redis down');
|
||||
};
|
||||
configureRateLimit({ default: 'redis', redis });
|
||||
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
expect(await claimCountedRateLimit(uniqueKey(), 1, WINDOW, 4)).toBe(4);
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
it('drops a memory counter once its windows have passed', async () => {
|
||||
configureRateLimit();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
windowStart(2_000);
|
||||
const key = uniqueKey();
|
||||
expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
|
||||
windowStart(2_002);
|
||||
sweepMemoryWindows();
|
||||
windowStart(2_000);
|
||||
expect(await claimCountedRateLimit(key, 1, WINDOW, 1)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('claimRateLimit', () => {
|
||||
const uniqueKey = () => `claim-${Math.random().toString(36).slice(2, 10)}`;
|
||||
|
||||
|
||||
@@ -290,26 +290,6 @@ describe('createRouteLifecycleMiddleware', () => {
|
||||
expect(after).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('answers 429 when the vetoing listener asks for it', async () => {
|
||||
const events = makeEvents();
|
||||
events.on(`${BASE}.before` as never, (_k, data) => {
|
||||
(data as RouteLifecycleEvent).allow = false;
|
||||
(data as RouteLifecycleEvent).rejectReason = 'slow down';
|
||||
(data as RouteLifecycleEvent).rejectStatus = 429;
|
||||
});
|
||||
const reject = record(events, `${BASE}.reject` as keyof EventMap);
|
||||
|
||||
const mw = createRouteLifecycleMiddleware(events, POST, PATH);
|
||||
const res = new FakeRes();
|
||||
await mw(makeReq('u-9'), res as unknown as Response, () => {});
|
||||
|
||||
expect(res.statusCode).toBe(429);
|
||||
expect(res.body).toMatchObject({
|
||||
error: { code: 'too_many_requests', message: 'slow down' },
|
||||
});
|
||||
expect(reject[0]).toMatchObject({ statusCode: 429 });
|
||||
});
|
||||
|
||||
it('treats a listener-sent response as a terminal after (not a reject) with the real status', async () => {
|
||||
const events = makeEvents();
|
||||
events.on(`${BASE}.before` as never, (_k, data) => {
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
import type { Request, RequestHandler } from 'express';
|
||||
import { actorUid } from '../actor';
|
||||
import type { EventClient } from '../../clients/event/EventClient';
|
||||
import type { LifecycleRejectStatus } from '../../clients/event/types';
|
||||
import type { RouteMethod } from './types';
|
||||
|
||||
type RoutePathValue = string | RegExp | Array<string | RegExp> | undefined;
|
||||
@@ -101,33 +100,30 @@ export const createRouteLifecycleMiddleware = (
|
||||
...base,
|
||||
allow: true as boolean,
|
||||
rejectReason: undefined as string | undefined,
|
||||
rejectStatus: undefined as LifecycleRejectStatus | undefined,
|
||||
};
|
||||
if (observeBefore) {
|
||||
await events.emitAndWait(beforeKey, beforeEvent, {});
|
||||
}
|
||||
|
||||
// Explicit veto: a listener set `allow = false`. Emit `reject` and
|
||||
// answer 403 (or the listener's 429) — unless the listener already
|
||||
// wrote its own response, in which case we report that status.
|
||||
// answer 403 — unless the listener already wrote its own response, in
|
||||
// which case we report that status rather than a misleading 403.
|
||||
if (beforeEvent.allow === false) {
|
||||
const status = beforeEvent.rejectStatus === 429 ? 429 : 403;
|
||||
events.emit(
|
||||
`${keyBase}.reject`,
|
||||
{
|
||||
phase: 'reject',
|
||||
...base,
|
||||
statusCode: res.headersSent ? res.statusCode : status,
|
||||
statusCode: res.headersSent ? res.statusCode : 403,
|
||||
durationMs: Date.now() - startedAt,
|
||||
rejectReason: beforeEvent.rejectReason,
|
||||
},
|
||||
{},
|
||||
);
|
||||
if (!res.headersSent) {
|
||||
res.status(status).json({
|
||||
res.status(403).json({
|
||||
error: {
|
||||
code:
|
||||
status === 429 ? 'too_many_requests' : 'forbidden',
|
||||
code: 'forbidden',
|
||||
message:
|
||||
beforeEvent.rejectReason ?? 'Blocked by policy',
|
||||
},
|
||||
|
||||
@@ -76,7 +76,6 @@ export type AccountGateUser = Partial<
|
||||
| 'phone'
|
||||
| 'card_fingerprint'
|
||||
| 'otp_enabled'
|
||||
| 'metadata'
|
||||
>
|
||||
> & { id?: number };
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { isHttpError } from './http/HttpError.js';
|
||||
import {
|
||||
SCHEDULE_LIST_LIMIT_CAP,
|
||||
SCHEDULE_PEER_PATH,
|
||||
schedulesUnavailable,
|
||||
} from './schedules.js';
|
||||
|
||||
describe('schedules contract', () => {
|
||||
it('answers an absent schedules service with a 503 events_disabled', () => {
|
||||
const err = schedulesUnavailable();
|
||||
expect(isHttpError(err)).toBe(true);
|
||||
expect(err.statusCode).toBe(503);
|
||||
expect(err.legacyCode).toBe('events_disabled');
|
||||
// Expected on a server without schedules, so it must not page.
|
||||
expect(err.noAlarm).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the list cap and the peer path the extension relies on', () => {
|
||||
expect(SCHEDULE_LIST_LIMIT_CAP).toBe(200);
|
||||
expect(SCHEDULE_PEER_PATH).toBe('/broadcast/events/schedules');
|
||||
});
|
||||
});
|
||||
@@ -1,188 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* What core asks of schedules. Nothing here is an implementation: an extension
|
||||
* registers `services.schedules`, and a server without one answers every
|
||||
* schedule route and `notify({ sendAt })` with `events_disabled`. Called by
|
||||
* extensions; keep these signatures stable.
|
||||
*/
|
||||
|
||||
import type { EventsDeployOutcome } from '../controllers/events/workerDeploy.js';
|
||||
import type { AppMessage } from '../services/notification/appMessage.js';
|
||||
import type { SuspendedReason } from '../stores/events/DurableSubscriptionStore.js';
|
||||
import type { DurableSubscription } from '../stores/events/types.js';
|
||||
import type { PageResult } from '../util/pagination.js';
|
||||
import type { Actor } from './actor.js';
|
||||
import { HttpError } from './http/HttpError.js';
|
||||
|
||||
/** Why a schedule ended. */
|
||||
export type ScheduleEndReason =
|
||||
| 'removed'
|
||||
| 'ended'
|
||||
| 'app_deleted'
|
||||
| 'uninstalled'
|
||||
| 'permission_revoked'
|
||||
| 'holder_gone'
|
||||
| 'app_gone'
|
||||
| 'orphaned'
|
||||
| 'delivered'
|
||||
| 'dropped';
|
||||
|
||||
/** What happened to a set of subscription rows schedules own. */
|
||||
export type ScheduleSubscriptionChange =
|
||||
| {
|
||||
type: 'suspended';
|
||||
reason: SuspendedReason;
|
||||
/** For a reason that never lifts; defaults to that reason. */
|
||||
endReason?: ScheduleEndReason;
|
||||
}
|
||||
| { type: 'resumed' }
|
||||
| { type: 'ended'; endReason: ScheduleEndReason };
|
||||
|
||||
/** A schedule that keeps running, one that runs once, or a delayed notification. */
|
||||
export type ScheduleViewKind = 'recurring' | 'once' | 'deferred';
|
||||
|
||||
/**
|
||||
* A schedule as its holder sees it. Times are epoch ms. `context` values are
|
||||
* never returned, as for a persistent subscription: only its keys and a hash.
|
||||
*/
|
||||
export interface ScheduleView {
|
||||
scheduleId: string;
|
||||
kind: ScheduleViewKind;
|
||||
appUid: string;
|
||||
/** `null` once ended: the handler binding goes with the schedule. */
|
||||
handlerName: string | null;
|
||||
contextKeys: string[] | null;
|
||||
contextHash: string | null;
|
||||
/** Interval of an `every` schedule. */
|
||||
every: number | null;
|
||||
cron: string | null;
|
||||
tz: string;
|
||||
/** First allowed run; a one-shot's only run. */
|
||||
startAt: number;
|
||||
/** Last allowed run, inclusive. */
|
||||
endsAt: number | null;
|
||||
/** `sending` once a delayed send has started; it can't be cancelled then. */
|
||||
state: 'active' | 'suspended' | 'sending' | 'ended';
|
||||
suspendedReason: string | null;
|
||||
endReason: string | null;
|
||||
endedAt: number | null;
|
||||
/**
|
||||
* The slot it runs for next; `null` while suspended or sending, and once
|
||||
* ended.
|
||||
*/
|
||||
nextRunAt: number | null;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
export interface ScheduleListRequest {
|
||||
limit: number;
|
||||
cursor?: string;
|
||||
includeTotal: boolean;
|
||||
}
|
||||
|
||||
/** The caps a holder's schedules are held to, from their plan. */
|
||||
export interface ScheduleQuota {
|
||||
recurringPerUser: number;
|
||||
recurringPerApp: number;
|
||||
oneShotsPerUser: number;
|
||||
oneShotsPerApp: number;
|
||||
/** Worst-case runs per hour × 1000, summed over recurring rows. */
|
||||
budgetPerUser: number;
|
||||
budgetPerApp: number;
|
||||
}
|
||||
|
||||
/** A caller cleared to supply `sendAt`, and when it asked for. */
|
||||
export interface DeferredSendPlan {
|
||||
sendAt: number;
|
||||
/** Due within 5 seconds, or past: sent now rather than stored. */
|
||||
immediate: boolean;
|
||||
holderUserId: number;
|
||||
appUid: string;
|
||||
handlerDepth: number;
|
||||
quota: ScheduleQuota;
|
||||
}
|
||||
|
||||
/** What a delayed app message stores until it is sent. */
|
||||
export interface DeferredAppMessage {
|
||||
message: AppMessage;
|
||||
/** A KV share handle naming the recipient. */
|
||||
to?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A condition on one row of another table that statements embed as `EXISTS
|
||||
* (SELECT 1 FROM <table> WHERE <where>)`, binding `values`.
|
||||
*/
|
||||
export interface SqlGuard {
|
||||
table: string;
|
||||
where: string;
|
||||
values: unknown[];
|
||||
}
|
||||
|
||||
/** Ceiling of a schedule listing page. */
|
||||
export const SCHEDULE_LIST_LIMIT_CAP = 200;
|
||||
|
||||
/** The signed peer route a region's schedule changes are sent to. */
|
||||
export const SCHEDULE_PEER_PATH = '/broadcast/events/schedules';
|
||||
|
||||
export const schedulesUnavailable = (): HttpError =>
|
||||
new HttpError(503, 'Schedules are not available on this server', {
|
||||
legacyCode: 'events_disabled',
|
||||
// The designed answer of a server without schedules, not a fault.
|
||||
noAlarm: true,
|
||||
});
|
||||
|
||||
export interface ScheduleServiceContract {
|
||||
/**
|
||||
* Keep schedules in step with a change to the subscription rows they own.
|
||||
* Never throws: the subscription change has already happened.
|
||||
*/
|
||||
onSubscriptionsChanged(
|
||||
rows: readonly DurableSubscription[],
|
||||
change: ScheduleSubscriptionChange,
|
||||
): Promise<void>;
|
||||
createFor(
|
||||
actor: Actor,
|
||||
body: Record<string, unknown>,
|
||||
): Promise<ScheduleView>;
|
||||
list(
|
||||
actor: Actor,
|
||||
request: ScheduleListRequest,
|
||||
): Promise<PageResult<ScheduleView>>;
|
||||
get(actor: Actor, scheduleId: unknown): Promise<ScheduleView>;
|
||||
remove(actor: Actor, scheduleId: unknown): Promise<void>;
|
||||
receivePeerChanges(body: unknown): Promise<{ applied: number }>;
|
||||
/** Checks a caller supplying `sendAt`; `events_disabled` when off. */
|
||||
prepareSendAt(actor: Actor, sendAt: unknown): Promise<DeferredSendPlan>;
|
||||
createDeferredAppMessage(
|
||||
plan: DeferredSendPlan,
|
||||
message: DeferredAppMessage,
|
||||
): Promise<{ scheduleId: string; sendAt: number }>;
|
||||
/**
|
||||
* Runs an events worker deploy under the schedules' deploy gate; `capped`
|
||||
* for a schedule run's. A passthrough when schedules are off.
|
||||
*/
|
||||
gateDeploy?(
|
||||
script: string,
|
||||
options: { capped: boolean },
|
||||
deploy: () => Promise<EventsDeployOutcome>,
|
||||
): Promise<EventsDeployOutcome>;
|
||||
}
|
||||
@@ -104,8 +104,6 @@ const default_user_permissions = {
|
||||
// is there; which app may is the per-app grant, which is what the user is
|
||||
// actually asked about.
|
||||
'events:background': {},
|
||||
// Likewise, the per-app grant decides whose notifications go out as pushes.
|
||||
'notifications:push': {},
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -209,7 +209,6 @@ afterAll(async () => {
|
||||
const attemptsFor = async (
|
||||
model: string,
|
||||
thrown: unknown = new Error('upstream down'),
|
||||
extra: { provider?: string; fallback?: boolean } = {},
|
||||
) => {
|
||||
createMock.mockRejectedValue(thrown);
|
||||
let caught: HttpError | undefined;
|
||||
@@ -218,7 +217,6 @@ const attemptsFor = async (
|
||||
driver.complete({
|
||||
model,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
...extra,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
@@ -888,59 +886,6 @@ describe('ChatCompletionDriver duplicate-model fallback', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('ChatCompletionDriver fallback opt-out', () => {
|
||||
const SHARED = 'deepseek-v4-pro';
|
||||
|
||||
it('fails on the first route when the caller sets fallback: false', async () => {
|
||||
const attempts = await attemptsFor(SHARED, undefined, {
|
||||
fallback: false,
|
||||
});
|
||||
|
||||
expect(attempts).toHaveLength(1);
|
||||
expect(attempts[0]).toMatchObject({ provider: 'deepseek' });
|
||||
});
|
||||
|
||||
it('tries only the pinned provider', async () => {
|
||||
const attempts = await attemptsFor(SHARED, undefined, {
|
||||
provider: 'openrouter',
|
||||
fallback: false,
|
||||
});
|
||||
|
||||
expect(attempts).toHaveLength(1);
|
||||
expect(attempts[0]).toMatchObject({ provider: 'openrouter' });
|
||||
});
|
||||
|
||||
it('rejects a pinned provider that has no route for the model', async () => {
|
||||
createMock.mockClear();
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
driver.complete({
|
||||
model: SHARED,
|
||||
provider: 'claude',
|
||||
fallback: false,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still marks the failed route for the next caller', async () => {
|
||||
const [only] = await attemptsFor(SHARED, undefined, {
|
||||
fallback: false,
|
||||
});
|
||||
|
||||
expect(isRouteUnhealthy(only.provider, only.model)).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the chain when fallback is true or unset', async () => {
|
||||
expect(
|
||||
await attemptsFor(SHARED, undefined, { fallback: true }),
|
||||
).toHaveLength(3);
|
||||
expect(await attemptsFor(SHARED)).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
||||
// Each attempt in a fallback chain is a whole completion at that model's
|
||||
// prices, so each one has to clear the credit gate on its own. The chain used
|
||||
// to re-check with a nominal 1-microcent amount, which any account with a
|
||||
|
||||
@@ -709,20 +709,6 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
// A pinned provider without a route for the model normally resolves
|
||||
// to another vendor; with `fallback: false` that is the one thing the
|
||||
// caller asked us not to do.
|
||||
if (
|
||||
args.fallback === false &&
|
||||
intendedProvider &&
|
||||
model.provider !== intendedProvider
|
||||
) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`Model ${args.model} is not served by provider ${intendedProvider}`,
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
|
||||
args.messages = normalizeMediaParts(normalize_messages(args.messages));
|
||||
if (args.tools) {
|
||||
@@ -960,15 +946,8 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
const bucketKey = model.id;
|
||||
const tried = new Set([routeId(model.provider!, model.id)]);
|
||||
let lastError: Error | null = e as Error;
|
||||
// With `fallback: false` the caller wants this route's answer or
|
||||
// its error, not a bill for another provider's attempt.
|
||||
const fallbackAllowed = args.fallback !== false;
|
||||
|
||||
while (
|
||||
fallbackAllowed &&
|
||||
lastError &&
|
||||
attempts.length < MAX_ATTEMPTS
|
||||
) {
|
||||
while (lastError && attempts.length < MAX_ATTEMPTS) {
|
||||
const fallback = this.#findFallback(bucketKey, tried);
|
||||
if (!fallback) break;
|
||||
|
||||
|
||||
@@ -809,11 +809,6 @@ describe('Alibaba official catalog', () => {
|
||||
expect(models.some((model) => model.id === 'qwen-image-fixture')).toBe(
|
||||
false,
|
||||
);
|
||||
const fixture = models.find(
|
||||
(model) => model.id === 'qwen-catalog-fixture',
|
||||
)!;
|
||||
expect(fixture.puterId).toBe('alibaba:qwen/qwen-catalog-fixture');
|
||||
expect(modelLookupNames(models)).toContain('qwen-catalog-fixture');
|
||||
const plus = models.find((model) => model.id === 'qwen-plus')!;
|
||||
const original = ALIBABA_MODELS.find(
|
||||
(model) => model.id === 'qwen-plus',
|
||||
@@ -914,115 +909,6 @@ describe('Alibaba official catalog', () => {
|
||||
expect(model.costs).toEqual({ tokens: 1_000_000 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['qwen3.9-plus-2027-01-01', 'alibaba:qwen/qwen3.9-plus-2027-01-01'],
|
||||
['kimi/kimi-k3', 'alibaba:kimi/kimi-k3'],
|
||||
[
|
||||
'deepseek-v4.1-flash-1017',
|
||||
'alibaba:deepseek/deepseek-v4.1-flash-1017',
|
||||
],
|
||||
['glm-5.3-prime', 'alibaba:zhipu/glm-5.3-prime'],
|
||||
['decision-model-preview', 'alibaba:qwen/decision-model-preview'],
|
||||
])(
|
||||
'gives uncurated model %s a puterId in the alibaba:<vendor>/<id> form',
|
||||
(id, puterId) => {
|
||||
const model = mapAlibabaCatalogModel({
|
||||
model: id,
|
||||
capabilities: ['TG'],
|
||||
})!;
|
||||
expect(model.puterId).toBe(puterId);
|
||||
},
|
||||
);
|
||||
|
||||
it('skips a vendor-prefixed spelling of a curated model', () => {
|
||||
const model = mapAlibabaCatalogModel({
|
||||
model: 'zhipu/glm-5.2',
|
||||
capabilities: ['TG', 'Reasoning'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Text'],
|
||||
response_modality: ['Text'],
|
||||
},
|
||||
});
|
||||
expect(model).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
'ASR',
|
||||
{
|
||||
capabilities: ['ASR'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Audio'],
|
||||
response_modality: ['Text'],
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'rerank',
|
||||
{
|
||||
capabilities: ['TR'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Text'],
|
||||
response_modality: ['Text'],
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'image generation',
|
||||
{
|
||||
capabilities: ['IG'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Text'],
|
||||
response_modality: ['Text', 'Image'],
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'realtime omni',
|
||||
{
|
||||
capabilities: ['Realtime-Omni'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Text', 'Image', 'Audio', 'Video'],
|
||||
response_modality: ['Text', 'Audio'],
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'TTS',
|
||||
{
|
||||
capabilities: ['TTS'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Audio'],
|
||||
response_modality: ['Text'],
|
||||
},
|
||||
},
|
||||
],
|
||||
])('rejects %s models that cannot serve chat completions', (_kind, raw) => {
|
||||
expect(
|
||||
mapAlibabaCatalogModel({
|
||||
model: 'non-chat-fixture',
|
||||
...(raw as AlibabaApiModel),
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('admits a vision model', () => {
|
||||
const model = mapAlibabaCatalogModel({
|
||||
model: 'vision-fixture',
|
||||
capabilities: ['VU'],
|
||||
inference_metadata: {
|
||||
request_modality: ['Text', 'Image'],
|
||||
response_modality: ['Text'],
|
||||
},
|
||||
})!;
|
||||
expect(model.modalities?.input).toEqual(['text', 'image']);
|
||||
});
|
||||
|
||||
it('does not admit an uncurated entry without capability or modality data', () => {
|
||||
expect(mapAlibabaCatalogModel({ model: 'mystery-model' })).toBeNull();
|
||||
expect(mapAlibabaCatalogModel({ model: 'qwen-plus' })).not.toBeNull();
|
||||
});
|
||||
|
||||
it('refuses inference with unverified catalog prices', async () => {
|
||||
seedPages();
|
||||
const { provider } = makeProvider();
|
||||
|
||||
@@ -82,23 +82,6 @@ const positiveLimit = (value: unknown): number | undefined =>
|
||||
? value
|
||||
: undefined;
|
||||
|
||||
const chatCapabilities = ['TG', 'Reasoning', 'VU'];
|
||||
|
||||
const curatedPuterIds = new Set(ALIBABA_MODELS.map((model) => model.puterId));
|
||||
|
||||
const inferPuterId = (id: string): string => {
|
||||
if (id.includes('/')) return `alibaba:${id}`;
|
||||
const family = id.split('-')[0];
|
||||
const sibling = ALIBABA_MODELS.find(
|
||||
(model) => model.puterId && model.id.split('-')[0] === family,
|
||||
);
|
||||
// Unprefixed feed ids are Model Studio's own family unless a curated
|
||||
// sibling points at a third-party org.
|
||||
const org =
|
||||
sibling?.puterId?.slice('alibaba:'.length).split('/')[0] ?? 'qwen';
|
||||
return `alibaba:${org}/${id}`;
|
||||
};
|
||||
|
||||
export const mapAlibabaCatalogModel = (
|
||||
raw: AlibabaApiModel,
|
||||
previous?: IChatModel,
|
||||
@@ -107,25 +90,16 @@ export const mapAlibabaCatalogModel = (
|
||||
const id = raw.model.trim();
|
||||
const existing =
|
||||
previous ?? ALIBABA_MODELS.find((model) => model.id === id);
|
||||
const input = raw.inference_metadata?.request_modality;
|
||||
const output = raw.inference_metadata?.response_modality;
|
||||
if (output && !output.includes('Text')) return null;
|
||||
if (
|
||||
raw.capabilities &&
|
||||
!raw.capabilities.some((capability) =>
|
||||
chatCapabilities.includes(capability),
|
||||
!output &&
|
||||
!existing &&
|
||||
!raw.capabilities?.some((capability) =>
|
||||
['TG', 'Reasoning', 'VU'].includes(capability),
|
||||
)
|
||||
)
|
||||
return null;
|
||||
if (input && !input.includes('Text')) return null;
|
||||
if (output && !output.includes('Text')) return null;
|
||||
if (!raw.capabilities && !input && !output && !existing) return null;
|
||||
let puterId = existing?.puterId;
|
||||
if (!puterId) {
|
||||
puterId = inferPuterId(id);
|
||||
// A feed entry spelling a curated model differently would duplicate
|
||||
// the curated entry's puterId and list the model twice.
|
||||
if (curatedPuterIds.has(puterId)) return null;
|
||||
}
|
||||
const catalogPricing: AlibabaCatalogTier[] = (raw.prices ?? []).map(
|
||||
(tier) => {
|
||||
// Abbreviated ranges do not specify whether k means 1000 or 1024 tokens.
|
||||
@@ -175,7 +149,6 @@ export const mapAlibabaCatalogModel = (
|
||||
return {
|
||||
...existing,
|
||||
id,
|
||||
puterId,
|
||||
name: raw.name ?? existing?.name ?? id,
|
||||
description: raw.description ?? existing?.description,
|
||||
aliases: existing?.aliases,
|
||||
|
||||
+22
-26
@@ -177,7 +177,6 @@ describe('OpenAiChatProvider model catalog', () => {
|
||||
expect(ids).toContain('gpt-5.2-2025-12-11');
|
||||
expect(ids).toContain('gpt-6-astra');
|
||||
expect(ids).toContain('gpt-6.1-sol');
|
||||
expect(ids).toContain('gpt-6.1-terra');
|
||||
expect(ids).toContain('gpt-6-sol');
|
||||
expect(ids).toContain('gpt-6-luna');
|
||||
});
|
||||
@@ -349,33 +348,30 @@ describe('OpenAiChatProvider.complete request shape', () => {
|
||||
expect(args.safety_identifier).toBe('puter-u42');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'gpt-6-astra',
|
||||
'gpt-6.1-sol',
|
||||
'gpt-6.1-terra',
|
||||
'gpt-6-sol',
|
||||
'gpt-6-luna',
|
||||
])('resolves the namespaced %s alias', async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
it.each(['gpt-6-astra', 'gpt-6.1-sol', 'gpt-6-sol', 'gpt-6-luna'])(
|
||||
'resolves the namespaced %s alias',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: `openai/${model}`,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
reasoning_effort: 'low',
|
||||
}),
|
||||
);
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: `openai/${model}`,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
reasoning_effort: 'low',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(model);
|
||||
expect(createMock.mock.calls[0]![0].reasoning_effort).toBe('low');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
`openai:${model}`,
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(model);
|
||||
expect(createMock.mock.calls[0]![0].reasoning_effort).toBe('low');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
`openai:${model}`,
|
||||
expect.any(Object),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('omits parallel_tool_calls/tool_choice when no tool is sent (Chat Completions rejects them alone)', async () => {
|
||||
const { provider } = makeProvider();
|
||||
|
||||
@@ -224,7 +224,6 @@ describe('OpenAiResponsesChatProvider model catalog', () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
['gpt-6.1-terra', '2026-06-01', '2026-10-26', 100, 5, 500],
|
||||
['gpt-6.1-sol', '2026-04-30', '2026-09-29', 200, 10, 1000],
|
||||
['gpt-6-sol', '2026-04-20', '2026-09-22', 200, 20, 1000],
|
||||
['gpt-6-luna', '2026-05-18', '2026-09-22', 10, 1, 50],
|
||||
@@ -245,7 +244,6 @@ describe('OpenAiResponsesChatProvider model catalog', () => {
|
||||
tokens: 1_000_000,
|
||||
prompt_tokens: input,
|
||||
cached_tokens: cached,
|
||||
cache_write_tokens: input * 1.25,
|
||||
completion_tokens: output,
|
||||
},
|
||||
context: 1_050_000,
|
||||
@@ -334,7 +332,6 @@ describe('OpenAiResponsesChatProvider.complete request shape', () => {
|
||||
'gpt-6-luna',
|
||||
'gpt-6-astra',
|
||||
'gpt-6.1-sol',
|
||||
'gpt-6.1-terra',
|
||||
'gpt-5.6',
|
||||
'openai/gpt-5.6',
|
||||
'openai/gpt-5.6-luna',
|
||||
@@ -370,7 +367,7 @@ describe('OpenAiResponsesChatProvider.complete request shape', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['gpt-5.6-luna', 'gpt-6-luna', 'gpt-6.1-terra'])(
|
||||
it.each(['gpt-5.6-luna', 'gpt-6-luna'])(
|
||||
'preserves sampling with reasoning disabled for %s',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
@@ -623,13 +620,7 @@ describe('OpenAiResponsesChatProvider.complete request shape', () => {
|
||||
expect(gpt41Args).not.toHaveProperty('verbosity');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'gpt-6-astra',
|
||||
'gpt-6.1-sol',
|
||||
'gpt-6.1-terra',
|
||||
'gpt-6-sol',
|
||||
'gpt-6-luna',
|
||||
])(
|
||||
it.each(['gpt-6-astra', 'gpt-6.1-sol', 'gpt-6-sol', 'gpt-6-luna'])(
|
||||
'maps flat controls to Responses options for %s aliases',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
@@ -869,51 +860,47 @@ describe('OpenAiResponsesChatProvider.complete non-stream output', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
['gpt-6-sol', 119],
|
||||
['gpt-6.1-terra', 59],
|
||||
] as const)(
|
||||
'bills %s at long-context rates past 272K input tokens',
|
||||
async (modelId, total) => {
|
||||
const model = OPEN_AI_MODELS.find((m) => m.id === modelId)!;
|
||||
const { provider } = makeProvider();
|
||||
responsesCreateMock.mockResolvedValueOnce({
|
||||
output: [{ role: 'assistant' }],
|
||||
output_text: 'hi',
|
||||
usage: {
|
||||
input_tokens: 300_000,
|
||||
output_tokens: 10_000,
|
||||
input_tokens_details: {
|
||||
cached_tokens: 50_000,
|
||||
cache_write_tokens: 20_000,
|
||||
},
|
||||
it('bills the whole request at long-context rates past 272K input tokens', async () => {
|
||||
const sol = OPEN_AI_MODELS.find((m) => m.id === 'gpt-6-sol')!;
|
||||
const { provider } = makeProvider();
|
||||
responsesCreateMock.mockResolvedValueOnce({
|
||||
output: [{ role: 'assistant' }],
|
||||
output_text: 'hi',
|
||||
usage: {
|
||||
input_tokens: 300_000,
|
||||
output_tokens: 10_000,
|
||||
input_tokens_details: {
|
||||
cached_tokens: 50_000,
|
||||
cache_write_tokens: 20_000,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: modelId,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'gpt-6-sol',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const [, , , overrides] = recordSpy.mock.calls[0]!;
|
||||
expect(overrides).toEqual({
|
||||
prompt_tokens: 230_000 * Number(model.costs.prompt_tokens) * 2,
|
||||
cached_tokens: 50_000 * Number(model.costs.cached_tokens) * 2,
|
||||
cache_write_tokens:
|
||||
20_000 * Number(model.costs.cache_write_tokens) * 2,
|
||||
completion_tokens:
|
||||
10_000 * Number(model.costs.completion_tokens) * 1.5,
|
||||
});
|
||||
const totalCents =
|
||||
Object.values(overrides as Record<string, number>).reduce(
|
||||
(a, b) => a + b,
|
||||
0,
|
||||
) / 1_000_000;
|
||||
expect(totalCents).toBeCloseTo(total);
|
||||
},
|
||||
);
|
||||
const [, , , overrides] = recordSpy.mock.calls[0]!;
|
||||
// $0.92 + $0.02 + $0.10 + $0.15 = $1.19, against $0.62 at the
|
||||
// standard rates.
|
||||
expect(overrides).toEqual({
|
||||
prompt_tokens: 230_000 * Number(sol.costs.prompt_tokens) * 2,
|
||||
cached_tokens: 50_000 * Number(sol.costs.cached_tokens) * 2,
|
||||
cache_write_tokens:
|
||||
20_000 * Number(sol.costs.cache_write_tokens) * 2,
|
||||
completion_tokens:
|
||||
10_000 * Number(sol.costs.completion_tokens) * 1.5,
|
||||
});
|
||||
const totalCents =
|
||||
Object.values(overrides as Record<string, number>).reduce(
|
||||
(a, b) => a + b,
|
||||
0,
|
||||
) / 1_000_000;
|
||||
expect(totalCents).toBeCloseTo(119);
|
||||
});
|
||||
|
||||
it('bills cached tokens at the input rate when the model prices no cache read', async () => {
|
||||
// gpt-5.4-pro is responses-API-only and its catalogue entry has no
|
||||
|
||||
@@ -31,31 +31,6 @@ const GPT_LONG_CONTEXT_PRICING = {
|
||||
|
||||
// Hardcoded from https://models.dev/api.json
|
||||
export const OPEN_AI_MODELS: IChatModel[] = [
|
||||
{
|
||||
puterId: 'openai:openai/gpt-6.1-terra',
|
||||
id: 'gpt-6.1-terra',
|
||||
responsesSampling: 'reasoningDisabled',
|
||||
modalities: { input: ['text', 'image'], output: ['text'] },
|
||||
open_weights: false,
|
||||
tool_call: true,
|
||||
knowledge: '2026-06-01',
|
||||
release_date: '2026-10-26',
|
||||
aliases: ['openai/gpt-6.1-terra'],
|
||||
costs_currency: 'usd-cents',
|
||||
input_cost_key: 'prompt_tokens',
|
||||
output_cost_key: 'completion_tokens',
|
||||
costs: {
|
||||
tokens: 1_000_000,
|
||||
prompt_tokens: 100,
|
||||
cached_tokens: 5,
|
||||
cache_write_tokens: 100 * 1.25,
|
||||
completion_tokens: 500,
|
||||
},
|
||||
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
|
||||
context: 1_050_000,
|
||||
max_tokens: 128_000,
|
||||
responses_api: true,
|
||||
},
|
||||
{
|
||||
puterId: 'openai:openai/gpt-6.1-sol',
|
||||
id: 'gpt-6.1-sol',
|
||||
|
||||
@@ -175,11 +175,6 @@ export interface UsageDetails {
|
||||
export interface ICompleteArguments {
|
||||
messages: PuterMessage[];
|
||||
provider?: string;
|
||||
/**
|
||||
* `false` makes the first route's failure final instead of retrying the
|
||||
* request on another provider that serves the same model.
|
||||
*/
|
||||
fallback?: boolean;
|
||||
stream?: boolean;
|
||||
model: string;
|
||||
test_mode?: boolean;
|
||||
|
||||
Loaded 100 of 633 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user