mirror of
https://github.com/HeyPuter/puter.git
synced 2026-11-13 20:12:57 -10:00
Compare commits
142
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
465d9b35d0 | ||
|
|
61b713438d | ||
|
|
7ceaf69651 | ||
|
|
c00f693ec6 | ||
|
|
dace77c212 | ||
|
|
8e1233d82a | ||
|
|
722c25e0dc | ||
|
|
9e449e635b | ||
|
|
ba7011b5bb | ||
|
|
25d5843d03 | ||
|
|
4e6d655f4c | ||
|
|
bd1dd6564e | ||
|
|
69a3e918a7 | ||
|
|
aded055212 | ||
|
|
e54750492f | ||
|
|
406fe4dcfc | ||
|
|
2e085c0b75 | ||
|
|
b95fb870a5 | ||
|
|
28be1851f9 | ||
|
|
ff83360272 | ||
|
|
b19fde242f | ||
|
|
c2b38786f7 | ||
|
|
7c6c0f2d39 | ||
|
|
5ac41f90b6 | ||
|
|
c024e007aa | ||
|
|
eeedeb817f | ||
|
|
ca79f01edd | ||
|
|
55ba45e159 | ||
|
|
e8688d5025 | ||
|
|
10f10029c2 | ||
|
|
11a94ed608 | ||
|
|
5c35f8d053 | ||
|
|
c5a4b1f3b5 | ||
|
|
179ad990d6 | ||
|
|
ff8dde6895 | ||
|
|
d587ffa428 | ||
|
|
0fe68dae30 | ||
|
|
eb1ad240f2 | ||
|
|
d40cc499f5 | ||
|
|
06ef9506fa | ||
|
|
5513c23042 | ||
|
|
8aade99cc8 | ||
|
|
6f9719fd65 | ||
|
|
352a2fbabd | ||
|
|
59700519a6 | ||
|
|
8482ff2167 | ||
|
|
b42f2acf06 | ||
|
|
1b6de39a13 | ||
|
|
1aa7caf44c | ||
|
|
40253ae12b | ||
|
|
3b4945f018 | ||
|
|
fe58213bb5 | ||
|
|
6c35780d59 | ||
|
|
415c179770 | ||
|
|
feb0937b6d | ||
|
|
f880544ebe | ||
|
|
25a574b57d | ||
|
|
389fd26bee | ||
|
|
d77ebadfe6 | ||
|
|
7e78a4feef | ||
|
|
7aa4d91ae9 | ||
|
|
5f7ab80721 | ||
|
|
b3dbfdbd7b | ||
|
|
60dfff3b0b | ||
|
|
5a83c4a045 | ||
|
|
fe8acf007c | ||
|
|
8010c9fc94 | ||
|
|
bbf375db14 | ||
|
|
fe70e7c6ba | ||
|
|
f0d7a69dd4 | ||
|
|
5ce131517b | ||
|
|
49b38f3482 | ||
|
|
241fb7be8a | ||
|
|
0167a9d10e | ||
|
|
5a879a4346 | ||
|
|
c13bff0563 | ||
|
|
2f94ee1088 | ||
|
|
3068059058 | ||
|
|
7ee0cbfac9 | ||
|
|
dfa6f4effc | ||
|
|
20c63b6282 | ||
|
|
dd44c0704a | ||
|
|
681b9cd463 | ||
|
|
9fb66137a3 | ||
|
|
057aa116a4 | ||
|
|
1cea1b2a0a | ||
|
|
6c78d2f9e9 | ||
|
|
e8aa12ace5 | ||
|
|
f1964778b4 | ||
|
|
c8f7b1ceda | ||
|
|
0411a92f4e | ||
|
|
5cfd4d1f53 | ||
|
|
e1a36215f6 | ||
|
|
6f13b1f457 | ||
|
|
996d85148e | ||
|
|
09da68424e | ||
|
|
7bccde40a8 | ||
|
|
cf5d2c60a5 | ||
|
|
b7a0276337 | ||
|
|
bbc056cc49 | ||
|
|
e2f6c2ed05 | ||
|
|
745b3a4318 | ||
|
|
2dfd93687f | ||
|
|
91554a137e | ||
|
|
52da7b571c | ||
|
|
9e1ccf7ff3 | ||
|
|
8f94514692 | ||
|
|
2ad49e1040 | ||
|
|
57349a6982 | ||
|
|
ac0de9df44 | ||
|
|
7d8711bf5b | ||
|
|
af13c33678 | ||
|
|
5cf13547f2 | ||
|
|
e181a25f95 | ||
|
|
5e376bc0c8 | ||
|
|
fceaee6372 | ||
|
|
47a6122c37 | ||
|
|
9fabd1d5b3 | ||
|
|
32b28c1994 | ||
|
|
21625de22e | ||
|
|
c79b3edf4f | ||
|
|
d366861321 | ||
|
|
f850155411 | ||
|
|
c8998075ea | ||
|
|
759f1a798d | ||
|
|
b57fef8c9d | ||
|
|
7c0cf6f6a1 | ||
|
|
7f93166560 | ||
|
|
21846192d1 | ||
|
|
b9f0c8601d | ||
|
|
68249028ef | ||
|
|
cbcbdc2599 | ||
|
|
3296c1b0d6 | ||
|
|
a10a9d9b93 | ||
|
|
ad5f9b6196 | ||
|
|
65b966c264 | ||
|
|
c701b8d7c0 | ||
|
|
4b41b9cd54 | ||
|
|
ce5f28e844 | ||
|
|
97beffb572 | ||
|
|
03eab894a0 | ||
|
|
53c77c8333 |
No files matched your search
@@ -350,6 +350,9 @@
|
||||
// not, a row past this is swept. Off by default; any value above 0 arms
|
||||
// the sweep.
|
||||
// "notificationRetentionDays": 14,
|
||||
//
|
||||
// Ms a live notification waits for open tabs to acknowledge it.
|
||||
// "notificationAckTimeoutMs": 5000,
|
||||
|
||||
// ── Alarms / alerting ───────────────────────────────────────────────
|
||||
// Where system alarms go. Severity is the routing decision — each
|
||||
|
||||
@@ -47,7 +47,7 @@ Both are supported ways to define an API. **Prefer a controller when you need fi
|
||||
|
||||
Provider sign-in requires the account's authenticator or a recovery code when 2FA is enabled, including signup callbacks that resolve an existing account. The five-minute challenge uses the same single-use `/login/otp` and `/login/recovery-code` verification as password sign-in. The GUI carries it in a URL fragment and removes it before initialization. A session and successful popup-return proof are issued only after verification; cancelling retains the popup's signed retry context without marking it authenticated.
|
||||
|
||||
Disabling 2FA through `/user-protected/disable-2fa` requires both password or OIDC revalidation and the current six-digit authenticator `code`. Missing, malformed, or incorrect codes leave the secret and recovery codes intact. A team's requirement for 2FA still prevents disabling it.
|
||||
Disabling 2FA through `/user-protected/disable-2fa` requires both password or OIDC revalidation and either the current six-digit authenticator `code` or an unused recovery code in `code`. Recovery codes are consumed atomically. Missing or incorrect codes leave the secret and remaining recovery codes intact. A team's requirement for 2FA still prevents disabling it.
|
||||
|
||||
Email changes through `/user-protected/change-email` require the password or OIDC revalidation cookie, plus a six-digit `code` from the authenticator when 2FA is enabled. Missing or malformed codes return `400 bad_request`; incorrect codes return `401 code_mismatch`. Both include `factor: 'otp'`. Confirmation links are bound to that verified request. A 2FA-enabled account must request a new link if its existing link was issued without second-factor verification.
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# Browser push client
|
||||
|
||||
Puter's desktop shows pushes through `/sw.js`. Core registers it, and shows Settings → Notifications, only when the `pushEnabled` gui param is `true`; core never sets it. A backend extension that delivers pushes sets it with `services.homepage.setGuiParam('pushEnabled', true)` and can load its own code into the worker with `services.homepage.registerServiceWorkerScript(path)`.
|
||||
|
||||
Core subscribes the browser to a push service only when the server also publishes `pushWebPushKey` (see [Built-in Web Push client](#built-in-web-push-client)), and it has no `/push/*` routes of its own. The device, app, preference and subscription endpoints below come from that extension. On a deployment without one, such as most self-hosted servers, they answer core's plain 404 (`code: 'not_found'`), which `puter.notifications` reports as `push_unavailable` and the Notifications tab shows as unavailable. A device the extension can't find is `device_not_found`.
|
||||
|
||||
## Push payload
|
||||
|
||||
`sw.js` reads the push's JSON body. The message is its `puter` field, or `data.puter` for services that only carry string data, as an object or a JSON string:
|
||||
|
||||
```ts
|
||||
{ uid?: string; title?: string; body?: string; icon?: string; group?: string;
|
||||
target?: { app?: string; path?: string }; data?: unknown; channel?: string;
|
||||
type?: 'challenge'; challenge?: string }
|
||||
```
|
||||
|
||||
`uid` is the notification's uid, a later push with the same `group` replaces the one showing, and `target` and `data` reach the app when the user taps it.
|
||||
|
||||
When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so the page's push client registers the new subscription.
|
||||
|
||||
## Registering
|
||||
|
||||
The registry is in [src/gui/src/helpers/pushClient.js](../src/gui/src/helpers/pushClient.js). A service script registers its client like this:
|
||||
|
||||
```js
|
||||
service_script(async (api) => {
|
||||
api.use()['push.registerPushClient']?.(client);
|
||||
});
|
||||
```
|
||||
|
||||
`use(id)` throws when nothing is registered under `id`, so the form above also runs on a Puter version without the registry and does nothing there. Any script that runs after the GUI bundle can call `use()['push.registerPushClient']?.(client)` the same way. Registering again replaces the client. Registering dispatches `push-client-registered` on `window`, and an open Notifications tab redraws.
|
||||
|
||||
With no client registered, the tab says push notifications aren't available in this browser.
|
||||
|
||||
### Built-in Web Push client
|
||||
|
||||
When the server also sets the `pushWebPushKey` gui param (a VAPID public key), core registers its own client ([webPushClient.js](../src/gui/src/helpers/webPushClient.js)) unless an extension registered one first. It uses `puter.notifications.enable()` / `disable()`, the same standard Web Push flow apps use on their own origins:
|
||||
|
||||
1. `GET /push/web/key` for the key, then `pushManager.subscribe()` on the `/sw.js` registration.
|
||||
2. `POST /push/web/subscriptions { subscription }`. A subscription the server already has answers `{ status: 'active' }`. A new one answers `{ status: 'pending' }`, and the server sends it a push with `{ type: 'challenge', challenge }`.
|
||||
3. The worker shows that push and posts `{ type: 'puter-push-challenge', challenge }` to the origin's windows; the page sends it to `POST /push/web/subscriptions/verify`. The device is registered once the page hands the confirmation back.
|
||||
4. `DELETE /push/web/subscriptions { endpoint }` removes it.
|
||||
|
||||
Apps on their own origin do the same through `push-sw.js`, which they import into their own worker. A subscription made with an app token is the app's: its pushes are that app's notifications only.
|
||||
|
||||
## Interface
|
||||
|
||||
```ts
|
||||
interface PushClient {
|
||||
enable(): Promise<{ deviceUid: string }>;
|
||||
disable(): Promise<void>;
|
||||
status(): Promise<{
|
||||
permission: 'default' | 'granted' | 'denied';
|
||||
registered: boolean;
|
||||
deviceUid?: string;
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. It subscribes with the `/sw.js` registration, registers the device with `POST /push/devices` (`platform: 'web'`), and resolves the device `uid` the server returned. It rejects when permission is refused or the device couldn't be registered.
|
||||
- `disable()` unsubscribes and removes this browser's device with `DELETE /push/devices/:uid`. The tab calls it for Turn off, and for Remove on the device marked as this browser; Remove on any other device is a plain `DELETE`.
|
||||
- `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user, and `deviceUid` is that device's `uid`.
|
||||
|
||||
After `enable()` resolves, the tab calls `POST /push/devices/:uid/test` and asks the user whether a notification showed up.
|
||||
|
||||
The tab also reads `GET /push/apps` and `GET`/`POST /push/preferences`.
|
||||
|
||||
Core keeps its own record of a browser the user turned off, `puter.push.browser-off:<user uuid>` in `localStorage`, written by Turn off and by Remove on this browser's device, and cleared when `enable()` resolves. The silent registration above skips a browser with that record, so a device removed here stays removed while one removed from another device, or a new sign-in in a browser that already allows notifications, registers again.
|
||||
@@ -314,6 +314,41 @@ describe('thumbnails extension — handleThumbnailRead', () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
it('keeps read URLs stable within an hour and rotates them at the next hour', async () => {
|
||||
const key = mintedKey();
|
||||
const signedAt = Date.UTC(2026, 9, 26, 10);
|
||||
const read = async () => {
|
||||
const entry: Record<string, unknown> = {
|
||||
thumbnail: `s3://${BUCKET}/${key}`,
|
||||
};
|
||||
await handleThumbnailRead(entry, {
|
||||
s3,
|
||||
s3Presign,
|
||||
bucketName: BUCKET,
|
||||
bucketEndpoint: BUCKET_ENDPOINT,
|
||||
fsEntry: server.stores.fsEntry,
|
||||
});
|
||||
return entry.thumbnail as string;
|
||||
};
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
try {
|
||||
vi.setSystemTime(signedAt + 1000);
|
||||
const first = await read();
|
||||
vi.setSystemTime(signedAt + 3599_000);
|
||||
expect(await read()).toBe(first);
|
||||
expect(new URL(first).searchParams.get('X-Amz-Date')).toBe(
|
||||
'20261026T100000Z',
|
||||
);
|
||||
expect(new URL(first).searchParams.get('X-Amz-Expires')).toBe(
|
||||
'604800',
|
||||
);
|
||||
vi.setSystemTime(signedAt + 3600_000);
|
||||
expect(await read()).not.toBe(first);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
// Thumbnails stored before keys were bound to entries keep displaying.
|
||||
it.each([
|
||||
['an entry-bound key', () => mintedKey()],
|
||||
|
||||
@@ -307,6 +307,9 @@ export const handleThumbnailRead = async (
|
||||
const thumb = entry.thumbnail;
|
||||
if (typeof thumb !== 'string' || !thumb) return;
|
||||
const presignClient = deps.s3Presign;
|
||||
const signingDate = new Date(
|
||||
Math.floor(Date.now() / 3_600_000) * 3_600_000,
|
||||
);
|
||||
|
||||
if (isStoragePointer(thumb, deps.bucketEndpoint)) {
|
||||
const key = resolveThumbnailKey(thumb);
|
||||
@@ -319,7 +322,7 @@ export const handleThumbnailRead = async (
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
{ expiresIn: 604800 },
|
||||
{ expiresIn: 604800, signingDate },
|
||||
);
|
||||
} else if (thumb.startsWith('data')) {
|
||||
// Move an inline thumbnail into storage. The row only takes the new
|
||||
@@ -373,7 +376,7 @@ export const handleThumbnailRead = async (
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
{ expiresIn: 604800 },
|
||||
{ expiresIn: 604800, signingDate },
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
Generated
+8
-8
@@ -15589,13 +15589,13 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/smtp-server": {
|
||||
"version": "3.19.23",
|
||||
"resolved": "https://registry.npmjs.org/smtp-server/-/smtp-server-3.19.23.tgz",
|
||||
"integrity": "sha512-t6AOBe7RR9Y/9Kox4e5SIm7wS2ZA8P0VuhVEZ5Yiox0WyevX+rMjATzIwEX3A+B6SGwnJ8QvoHtJAhbJYbjy3A==",
|
||||
"version": "3.19.24",
|
||||
"resolved": "https://registry.npmjs.org/smtp-server/-/smtp-server-3.19.24.tgz",
|
||||
"integrity": "sha512-alzRmwDr5ywR8xo6F2AXqTMWSEjhjC+qB/yHD+zcn2PSEpsJTJ/5O7QdDT4pGVvNNwE+1jhA1ax+hMwleTceWw==",
|
||||
"license": "MIT-0",
|
||||
"dependencies": {
|
||||
"ipv6-normalize": "1.0.1",
|
||||
"nodemailer": "10.1.2",
|
||||
"nodemailer": "10.1.4",
|
||||
"punycode.js": "2.3.1"
|
||||
},
|
||||
"engines": {
|
||||
@@ -15603,9 +15603,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/smtp-server/node_modules/nodemailer": {
|
||||
"version": "10.1.2",
|
||||
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.2.tgz",
|
||||
"integrity": "sha512-rnQn9Rzh8RAwcp42MVsIvxbsv6YZ2a2/av+uyJ/LPU1+2QSlrRgFsWWZTKenyRUqkIl8urOI0c4+CEcmFdsvQw==",
|
||||
"version": "10.1.4",
|
||||
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.4.tgz",
|
||||
"integrity": "sha512-EsMnOR2oRv4LGnWvHgcVYkkQ4O2j8KO4WTw6l+PHR4sOXo8SAgIraxtEnGDmotjbhv0hDsu3IuDIFZm8iXwO4A==",
|
||||
"license": "MIT-0",
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
@@ -17724,7 +17724,7 @@
|
||||
"postal-mime": "^3.0.1",
|
||||
"replicate": "^1.0.0",
|
||||
"sharp": "^0.35.5",
|
||||
"smtp-server": "^3.19.23",
|
||||
"smtp-server": "^3.19.24",
|
||||
"socket.io": "^4.8.4",
|
||||
"svg-captcha": "^1.4.0",
|
||||
"together-ai": "^0.33.0",
|
||||
|
||||
@@ -108,6 +108,25 @@ describe('AbstractDatabaseClient — unimplemented surface', () => {
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — replica-aware reads', () => {
|
||||
it('avoids a primary read when the replica has rows in sequential mode', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
expect(
|
||||
await client.tryHardRead('SELECT 1', [], { race: false }),
|
||||
).toEqual([{ id: 1 }]);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([[], new Error('replica unavailable')])(
|
||||
'falls back once in sequential mode',
|
||||
async (replica) => {
|
||||
const client = new FixtureDatabaseClient(replica, [{ id: 2 }]);
|
||||
expect(
|
||||
await client.tryHardRead('SELECT 1', [], { race: false }),
|
||||
).toEqual([{ id: 2 }]);
|
||||
expect(client.preads).toEqual(['SELECT 1']);
|
||||
},
|
||||
);
|
||||
|
||||
it('returns replica rows without waiting on the primary', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
|
||||
@@ -138,25 +138,27 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
|
||||
/**
|
||||
* Like `read()`, but an empty or failed replica read falls back to the
|
||||
* primary, so replication lag can't hide a row that was just written. The
|
||||
* primary read starts alongside the replica one rather than after it.
|
||||
* Without a replica this is a single `read()`.
|
||||
* primary, so replication lag can't hide a row that was just written. By
|
||||
* default the primary read starts alongside the replica one; `race: false`
|
||||
* reads the primary only after the replica returns empty or fails. Without
|
||||
* a replica this is a single `read()`.
|
||||
*/
|
||||
@Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
|
||||
async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
{ race = true }: { race?: boolean } = {},
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (!this.hasReadReplica()) return this.read(query, params);
|
||||
|
||||
const primary = settle(this.pread(query, params));
|
||||
const primary = race ? settle(this.pread(query, params)) : null;
|
||||
try {
|
||||
const rows = await this.replicaRead(query, params);
|
||||
if (rows.length > 0) return rows;
|
||||
} catch {
|
||||
// replica failed — fall through to primary
|
||||
}
|
||||
const result = await primary;
|
||||
const result = await (primary ?? settle(this.pread(query, params)));
|
||||
if ('error' in result) throw result.error;
|
||||
return result.value;
|
||||
}
|
||||
|
||||
@@ -217,7 +217,8 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
return this.configuration === Configuration.REPLICA;
|
||||
}
|
||||
|
||||
// No failover: `tryHardRead` already has a primary read in flight.
|
||||
// No failover: `tryHardRead` falls back to the primary on an empty result
|
||||
// or error (raced alongside by default, after the replica with `race: false`).
|
||||
protected override async replicaRead(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
|
||||
@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
|
||||
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
|
||||
|
||||
/** Highest schema version the migration table can reach. */
|
||||
const CURRENT_SCHEMA_VERSION = 91;
|
||||
const CURRENT_SCHEMA_VERSION = 93;
|
||||
|
||||
/**
|
||||
* These suites migrate real files on disk. Idle they finish in well under a
|
||||
|
||||
@@ -130,6 +130,8 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
|
||||
[88, ['0093_fsentries-parent-uid-index.sql']],
|
||||
[89, ['0094_deleted-apps.sql']],
|
||||
[90, ['0095_kv-share-handle-levels.sql']],
|
||||
[91, ['0096_kv-share-handle-quota-indexes.sql']],
|
||||
[92, ['0097_user-to-user-holder-permission-index.sql']],
|
||||
];
|
||||
|
||||
export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Share-handle quota, listing and purge indexes. See
|
||||
-- sqlite/0096_kv-share-handle-quota-indexes.sql for what each serves. No
|
||||
-- per-file applied-state tracking, so every index is guarded and a replay does
|
||||
-- nothing.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_kv_share_handle_quota_indexes;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_kv_share_handle_quota_indexes()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_owner_live'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_owner_live` (`owner_user_id`, `revoked_at`, `app_uid`);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_owner_app'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_owner_app` (`owner_user_id`, `app_uid`, `id`);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_grantee_app'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_grantee_app` (`grantee_user_id`, `app_uid`, `id`);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_pair'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_pair` (`owner_user_id`, `grantee_user_id`, `revoked_at`);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_revoked'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_revoked` (`revoked_at`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_kv_share_handle_quota_indexes();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_kv_share_handle_quota_indexes;
|
||||
@@ -0,0 +1,38 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Lets a holder's grant check read only the permissions it names. Guarded, so
|
||||
-- a replay does nothing.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_u2u_holder_permission_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_u2u_holder_permission_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user_to_user_permissions'
|
||||
AND INDEX_NAME = 'idx_user_to_user_permissions_holder_permission'
|
||||
) THEN
|
||||
ALTER TABLE `user_to_user_permissions` ADD INDEX `idx_user_to_user_permissions_holder_permission` (`holder_user_id`, `permission`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_u2u_holder_permission_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_u2u_holder_permission_index;
|
||||
@@ -0,0 +1,35 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Share-handle quota, listing and purge indexes. See
|
||||
-- sqlite/0096_kv-share-handle-quota-indexes.sql for what each serves.
|
||||
-- Idempotent via IF NOT EXISTS; there is no per-file applied-state tracking.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_owner_live
|
||||
ON kv_share_handles (owner_user_id, revoked_at, app_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_owner_app
|
||||
ON kv_share_handles (owner_user_id, app_uid, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_grantee_app
|
||||
ON kv_share_handles (grantee_user_id, app_uid, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_pair
|
||||
ON kv_share_handles (owner_user_id, grantee_user_id, revoked_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_revoked
|
||||
ON kv_share_handles (revoked_at);
|
||||
@@ -0,0 +1,21 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Lets a holder's grant check read only the permissions it names.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_to_user_permissions_holder_permission
|
||||
ON user_to_user_permissions (holder_user_id, permission);
|
||||
@@ -0,0 +1,42 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Indexes behind the share-handle quotas, the app-filtered listings and the
|
||||
-- retention purge on `kv_share_handles`.
|
||||
--
|
||||
-- - owner_live : the owner's live count, in total and per app, read off
|
||||
-- the index alone.
|
||||
-- - owner_app : the owner's listing narrowed to one app, in id order.
|
||||
-- - grantee_app : the holder's listing narrowed to one app, in id order.
|
||||
-- - pair : live handles one owner holds out to one holder, counted at
|
||||
-- mint and walked on revoke and unshare.
|
||||
-- - revoked : retired rows past retention, for the purge.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_owner_live`
|
||||
ON `kv_share_handles` (`owner_user_id`, `revoked_at`, `app_uid`);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_owner_app`
|
||||
ON `kv_share_handles` (`owner_user_id`, `app_uid`, `id`);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_grantee_app`
|
||||
ON `kv_share_handles` (`grantee_user_id`, `app_uid`, `id`);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_pair`
|
||||
ON `kv_share_handles` (`owner_user_id`, `grantee_user_id`, `revoked_at`);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_revoked`
|
||||
ON `kv_share_handles` (`revoked_at`);
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Lets a holder's grant check read only the permissions it names.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_user_to_user_permissions_holder_permission`
|
||||
ON `user_to_user_permissions` (`holder_user_id`, `permission`);
|
||||
@@ -534,6 +534,8 @@ export class DDBClient extends PuterClient {
|
||||
scanIndexForward?: boolean;
|
||||
beginsWith?: { key: string; value: string };
|
||||
select?: 'COUNT';
|
||||
/** Attributes to return instead of the whole item. */
|
||||
projection?: string[];
|
||||
filter?: {
|
||||
expression: string;
|
||||
values?: Record<string, unknown>;
|
||||
@@ -574,6 +576,10 @@ export class DDBClient extends PuterClient {
|
||||
Object.assign(expressionAttributeValues, options.filter.values);
|
||||
Object.assign(expressionAttributeNames, options.filter.names);
|
||||
}
|
||||
const projection = (options?.projection ?? []).map((name) => {
|
||||
expressionAttributeNames[`#p_${name}`] = name;
|
||||
return `#p_${name}`;
|
||||
});
|
||||
|
||||
const command = new QueryCommand({
|
||||
TableName: table,
|
||||
@@ -588,6 +594,9 @@ export class DDBClient extends PuterClient {
|
||||
? { FilterExpression: options.filter.expression }
|
||||
: {}),
|
||||
...(options?.select ? { Select: options.select } : {}),
|
||||
...(projection.length > 0
|
||||
? { ProjectionExpression: projection.join(', ') }
|
||||
: {}),
|
||||
...(options?.scanIndexForward !== undefined
|
||||
? { ScanIndexForward: options.scanIndexForward }
|
||||
: {}),
|
||||
|
||||
@@ -137,6 +137,17 @@ export type EventMap = {
|
||||
};
|
||||
'app.new-icon': { app_uid: string; data_url: string };
|
||||
'app.opened': { app_uid: string; user_id: number; ts: number };
|
||||
/**
|
||||
* A user removed an app: its grants and worker sessions are gone. With
|
||||
* `deleteData`, the user asked for the app's data in their account to go
|
||||
* too. Awaited by the emitter.
|
||||
*/
|
||||
'app.uninstalled': {
|
||||
userId: number;
|
||||
userUuid: string;
|
||||
appUid: string;
|
||||
deleteData: boolean;
|
||||
};
|
||||
'app.rename': {
|
||||
app_uid: string;
|
||||
old_name: string;
|
||||
@@ -197,7 +208,7 @@ export type EventMap = {
|
||||
*/
|
||||
is_temp?: boolean;
|
||||
ip?: string | null;
|
||||
source?: 'oidc';
|
||||
source?: 'oidc' | 'team_provision';
|
||||
req?: unknown;
|
||||
data?: unknown;
|
||||
abuse?: unknown;
|
||||
@@ -257,7 +268,7 @@ export type EventMap = {
|
||||
ip?: string | null;
|
||||
fingerprint?: string | null;
|
||||
reputation?: number | null;
|
||||
source?: 'oidc';
|
||||
source?: 'oidc' | 'team_provision';
|
||||
requires_phone_verification: boolean;
|
||||
requires_card_verification: boolean;
|
||||
accepted: boolean;
|
||||
@@ -389,6 +400,9 @@ export type EventMap = {
|
||||
new_username: string;
|
||||
};
|
||||
'user.email-changed': { user_id: number; new_email: string };
|
||||
/** An account was suspended, or its suspension lifted. Local events. */
|
||||
'user.suspended': { userId: number };
|
||||
'user.unsuspended': { userId: number };
|
||||
// Fired after an account is torn down (self-serve, admin, or temp-user
|
||||
// logout cleanup). Listeners purge external state tied to the account —
|
||||
// e.g. the marketplace extension cancels the user's Stripe subscriptions.
|
||||
@@ -538,6 +552,23 @@ export type EventMap = {
|
||||
/** Events handler runs behind whatever created it. */
|
||||
handlerDepth?: number;
|
||||
};
|
||||
/**
|
||||
* Every non-silent write or update, only on the node that wrote it. The
|
||||
* value is the listener's own copy of what the socket carries.
|
||||
*/
|
||||
'notif.written': {
|
||||
userId: number;
|
||||
uid: string;
|
||||
type: string;
|
||||
audience: string;
|
||||
/** The row's app; null when it names none or an update could not tell. */
|
||||
appUid: string | null;
|
||||
value: Record<string, unknown>;
|
||||
/** An earlier notification rewritten in place. */
|
||||
updated?: true;
|
||||
/** Names this one send, for a receipt asked about it. */
|
||||
sendId: string;
|
||||
};
|
||||
/**
|
||||
* One notification, addressed at whatever sockets each region holds for the
|
||||
* recipient. `outer.*` reaches peer regions and never sibling nodes, so
|
||||
@@ -549,6 +580,28 @@ export type EventMap = {
|
||||
wire: 'notif.message' | 'notif.unreads' | 'notif.ack';
|
||||
response: unknown;
|
||||
};
|
||||
/**
|
||||
* How one live notification went, on the node that asked for its receipt:
|
||||
* from the desktop's sockets, or with `appUid` the app's own.
|
||||
*/
|
||||
'notif.delivery.settled': {
|
||||
sendId?: string;
|
||||
uid: string;
|
||||
userId: number;
|
||||
type: string;
|
||||
/** The app whose own sockets were asked; absent for the desktop's. */
|
||||
appUid?: string;
|
||||
outcome: NotifDeliveryOutcome;
|
||||
/** A socket in this region acknowledges the receipt. */
|
||||
ackCapable: boolean;
|
||||
/** No socket here is an older client that acknowledges nothing. */
|
||||
everySocketAcks: boolean;
|
||||
/** Every node here answered the ask, not just this one. */
|
||||
complete: boolean;
|
||||
/** A peer region holds a socket for the user. */
|
||||
peerRegions: boolean;
|
||||
responders: NotifDeliveryResponder[];
|
||||
};
|
||||
|
||||
// ---- Metering ----
|
||||
// Recurring charges are pure mechanism here: the metering service knows
|
||||
@@ -665,6 +718,13 @@ export type EventMap = {
|
||||
* reason as the events generation bump: a per-process cache.
|
||||
*/
|
||||
'outer.pubsub.share.ownerShared': { ownerUserId: number };
|
||||
/**
|
||||
* Share handles were retired or changed, so every process must drop its
|
||||
* cached copy. `outer.pubsub.*` because the cache is a per-process map.
|
||||
*/
|
||||
'outer.pubsub.kv.share.forget': { handles: string[] };
|
||||
/** The same, for every cached handle one owner holds out. */
|
||||
'outer.pubsub.kv.share.forgetOwner': { ownerUserId: number };
|
||||
'outer.fs.write-hash': { hash: string; uuid: string };
|
||||
/**
|
||||
* Cache keys the KV read cache must stop serving, because the entries
|
||||
@@ -690,7 +750,7 @@ export type EventMap = {
|
||||
// ---- Subdomains ----
|
||||
// `uid` is the row uuid — for `delete` it is the only surviving handle a
|
||||
// listener can use to find state keyed to the row after it is gone.
|
||||
'subdomain.delete': { subdomain: string; uid?: string };
|
||||
'subdomain.delete': { subdomain: string; uid?: string; userId: number };
|
||||
'subdomain.update': { subdomain: string; uid?: string };
|
||||
'site.htmlServed': {
|
||||
subdomain: string;
|
||||
@@ -949,11 +1009,10 @@ export type EventKey = keyof EventMap & string;
|
||||
// Generates a wildcard for every non-final dot-separated prefix of K.
|
||||
export type WildcardPrefixes<K extends string> =
|
||||
K extends `${infer Head}.${infer Tail}`
|
||||
?
|
||||
| `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
? | `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
: never;
|
||||
|
||||
export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
|
||||
@@ -962,6 +1021,15 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
|
||||
? Extract<EventKey, `${Prefix}.${string}`>
|
||||
: P & EventKey;
|
||||
|
||||
/** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
|
||||
export type NotifDeliveryOutcome =
|
||||
'received' | 'unreceived' | 'no-socket' | 'remote-only';
|
||||
|
||||
/** One socket's receipt for a live notification. */
|
||||
export interface NotifDeliveryResponder {
|
||||
visibilityState: 'visible' | 'hidden' | 'unknown';
|
||||
}
|
||||
|
||||
export type EventMetadata = {
|
||||
from_outside?: boolean;
|
||||
/** Set by `emitAndWait` when a listener threw, so a verdict hook can refuse. */
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
describe('auth elevation cookies over HTTP', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
it.each(['bearer', 'cookie'])(
|
||||
'API host returns a usable token without setting cookies (%s auth)',
|
||||
async (credential) => {
|
||||
const { token, password, username } = env.users.user;
|
||||
const res = await fetch(new URL('/auth/elevate', env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(credential === 'cookie'
|
||||
? { cookie: `puter_auth_token=${token}` }
|
||||
: { authorization: `Bearer ${token}` }),
|
||||
},
|
||||
body: JSON.stringify({ password }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get('set-cookie')).toBeNull();
|
||||
const body = (await res.json()) as {
|
||||
elevated: boolean;
|
||||
token: string;
|
||||
};
|
||||
expect(body.elevated).toBe(true);
|
||||
expect(typeof body.token).toBe('string');
|
||||
const user = await env.server.stores.user.getByUsername(username);
|
||||
const { verifyStepUpSession } =
|
||||
await import('../../core/http/middleware/stepUpSession.js');
|
||||
expect(
|
||||
verifyStepUpSession(
|
||||
{
|
||||
headers: { 'x-puter-elevation': body.token },
|
||||
actor: { user: { uuid: user!.uuid } },
|
||||
} as never,
|
||||
{ tokenService: env.server.services.token },
|
||||
),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('GUI host sets the elevation cookie and omits the raw token for cookie auth', async () => {
|
||||
const { token, password } = env.users.user;
|
||||
const res = await fetch(new URL('/auth/elevate', env.origin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
cookie: `puter_auth_token=${token}`,
|
||||
},
|
||||
body: JSON.stringify({ password }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get('set-cookie')).toContain('puter_elevated=');
|
||||
expect(await res.json()).toEqual({ elevated: true });
|
||||
});
|
||||
});
|
||||
@@ -50,7 +50,10 @@ import {
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { createSecret as otpCreateSecret } from '../../services/auth/OTPUtil.js';
|
||||
import { kvSharePermission } from '../../services/kvShare/kvShares.js';
|
||||
import { FULL_API_ACCESS } from '../../services/permission/consts.js';
|
||||
import {
|
||||
FULL_API_ACCESS,
|
||||
NOTIFICATIONS_PUSH_PERMISSION,
|
||||
} from '../../services/permission/consts.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { resetCardVerificationStatusCache } from '../../util/cardFallback.js';
|
||||
import { FS_READ_LIMIT } from '../fs/limits.js';
|
||||
@@ -2179,6 +2182,78 @@ describe('AuthController grant flows', () => {
|
||||
).toContain(permission);
|
||||
});
|
||||
|
||||
it('grant-user-app: grants and revokes notifications:push for an app', async () => {
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `tgp-${uuidv4()}`,
|
||||
title: 'TestGrantPushApp',
|
||||
index_url: 'https://example.test/index.html',
|
||||
},
|
||||
{ ownerUserId: issuer.id },
|
||||
);
|
||||
const user = await server.stores.user.getById(issuer.id);
|
||||
const appActor = makeActor({
|
||||
user: user!,
|
||||
app: { uid: app.uid, id: app.id },
|
||||
});
|
||||
const hasPush = () =>
|
||||
server.services.permission.check(
|
||||
appActor,
|
||||
NOTIFICATIONS_PUSH_PERMISSION,
|
||||
);
|
||||
expect(await hasPush()).toBe(false);
|
||||
|
||||
// Anything past the bare permission names nothing a prompt describes.
|
||||
await expect(
|
||||
inCtx(issuerActor, () =>
|
||||
controller.handleGrantUserApp(
|
||||
makeReq(
|
||||
{
|
||||
app_uid: app.uid,
|
||||
permission: `${NOTIFICATIONS_PUSH_PERMISSION}:extra`,
|
||||
},
|
||||
{ actor: issuerActor },
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'permission_not_grantable',
|
||||
});
|
||||
expect(await hasPush()).toBe(false);
|
||||
|
||||
const res = makeRes();
|
||||
await inCtx(issuerActor, () =>
|
||||
controller.handleGrantUserApp(
|
||||
makeReq(
|
||||
{
|
||||
app_uid: app.uid,
|
||||
permission: NOTIFICATIONS_PUSH_PERMISSION,
|
||||
},
|
||||
{ actor: issuerActor },
|
||||
),
|
||||
res,
|
||||
),
|
||||
);
|
||||
expect(res.body).toEqual({});
|
||||
expect(await hasPush()).toBe(true);
|
||||
|
||||
await inCtx(issuerActor, () =>
|
||||
controller.handleRevokeUserApp(
|
||||
makeReq(
|
||||
{
|
||||
app_uid: app.uid,
|
||||
permission: NOTIFICATIONS_PUSH_PERMISSION,
|
||||
},
|
||||
{ actor: issuerActor },
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
);
|
||||
expect(await hasPush()).toBe(false);
|
||||
});
|
||||
|
||||
it('grant-user-app: resolves `origin` to the app when app_uid is omitted', async () => {
|
||||
// The GUI permission dialog and puter.perms.grantOrigin() identify
|
||||
// third-party sites by origin rather than app uid.
|
||||
@@ -7716,6 +7791,12 @@ describe('AuthController.handleCheckPermissions + handleListPermissions', () =>
|
||||
grant,
|
||||
),
|
||||
);
|
||||
// What launching the app records for the holder.
|
||||
await server.services.permission.grantUserAppPermission(
|
||||
holderActor,
|
||||
app.uid,
|
||||
'flag:app-is-authenticated',
|
||||
);
|
||||
const appActor = makeActor({
|
||||
user: holderActor.user,
|
||||
app: { id: app.id, uid: app.uid },
|
||||
@@ -9658,6 +9739,70 @@ describe('AuthController grant/revoke additional branches', () => {
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('revoke-user-app: 400 on `deleteData` without "*" or as a non-boolean', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `tr-delete-data-${uuidv4()}`,
|
||||
title: 'TestDeleteDataApp',
|
||||
index_url: `https://tr-delete-data-${uuidv4()}.example.test/`,
|
||||
},
|
||||
{ ownerUserId: actor.user.id },
|
||||
);
|
||||
for (const body of [
|
||||
{ permission: 'events:background', deleteData: true },
|
||||
{ permission: '*', deleteData: 'yes' },
|
||||
]) {
|
||||
await expect(
|
||||
controller.handleRevokeUserApp(
|
||||
makeReq({ app_uid: app.uid, ...body }, { actor }),
|
||||
makeRes(),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
}
|
||||
});
|
||||
|
||||
it('revoke-user-app `*`: announces the uninstall, with `deleteData` as sent', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `tr-uninstall-${uuidv4()}`,
|
||||
title: 'TestUninstallApp',
|
||||
index_url: `https://tr-uninstall-${uuidv4()}.example.test/`,
|
||||
},
|
||||
{ ownerUserId: actor.user.id },
|
||||
);
|
||||
const seen: unknown[] = [];
|
||||
const listener = (_key: string, data: unknown) => {
|
||||
seen.push(data);
|
||||
};
|
||||
server.clients.event.on('app.uninstalled', listener);
|
||||
try {
|
||||
for (const deleteData of [undefined, true]) {
|
||||
await inCtx(actor, () =>
|
||||
controller.handleRevokeUserApp(
|
||||
makeReq(
|
||||
{ app_uid: app.uid, permission: '*', deleteData },
|
||||
{ actor },
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
server.clients.event.off('app.uninstalled', listener);
|
||||
}
|
||||
const expected = {
|
||||
userId: actor.user.id,
|
||||
userUuid: actor.user.uuid,
|
||||
appUid: app.uid,
|
||||
};
|
||||
expect(seen).toEqual([
|
||||
{ ...expected, deleteData: false },
|
||||
{ ...expected, deleteData: true },
|
||||
]);
|
||||
});
|
||||
|
||||
it('revoke-user-app: 400 when permission is "*" but app_uid is missing', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
await expect(
|
||||
|
||||
@@ -3327,7 +3327,8 @@ export class AuthController extends PuterController {
|
||||
})
|
||||
async handleRevokeUserApp(req: Request, res: Response): Promise<void> {
|
||||
let { app_uid } = req.body ?? {};
|
||||
const { origin, permission, permissions, meta } = req.body ?? {};
|
||||
const { origin, permission, permissions, meta, deleteData } =
|
||||
req.body ?? {};
|
||||
this.#validateAppPermissionParams({
|
||||
app_uid,
|
||||
origin,
|
||||
@@ -3343,20 +3344,25 @@ export class AuthController extends PuterController {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
if (deleteData !== undefined && typeof deleteData !== 'boolean') {
|
||||
throw new HttpError(400, '`deleteData` must be a boolean', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
if (deleteData && permission !== '*') {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'`deleteData` only applies when `permission` is "*"',
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
// Deliberately not gated by the target's sharing flag: a user must
|
||||
// always be able to withdraw a grant, whatever the target now says.
|
||||
if (permission === '*') {
|
||||
await this.services.permission.revokeUserAppAll(
|
||||
req.actor!,
|
||||
app_uid,
|
||||
meta ?? undefined,
|
||||
);
|
||||
// Withdrawing everything also retires the workers the app deployed
|
||||
// under this user; their tokens would otherwise act as the app.
|
||||
await this.services.auth.deleteWorkerSessionsForApp(
|
||||
req.actor!.user!.id as number,
|
||||
app_uid,
|
||||
);
|
||||
await this.services.auth.uninstallApp(req.actor!, app_uid, {
|
||||
deleteData: deleteData === true,
|
||||
meta: meta ?? undefined,
|
||||
});
|
||||
} else {
|
||||
for (const entry of list) {
|
||||
await this.services.permission.revokeUserAppPermission(
|
||||
@@ -4269,7 +4275,27 @@ export class AuthController extends PuterController {
|
||||
);
|
||||
}
|
||||
|
||||
this.#requireAccountTwoFactor(user, req.body?.code);
|
||||
const code = req.body?.code;
|
||||
if (
|
||||
user.otp_enabled &&
|
||||
typeof code === 'string' &&
|
||||
/^[A-Z0-9]{8}$/.test(code)
|
||||
) {
|
||||
await this.#assertSecondFactorAttemptsRemain(user.uuid);
|
||||
const consumed = await this.stores.user.consumeOtpRecoveryCode(
|
||||
user.id,
|
||||
hashRecoveryCode(code),
|
||||
);
|
||||
if (!consumed) {
|
||||
await this.#recordSecondFactorFailure(user.uuid);
|
||||
throw new HttpError(401, 'Incorrect code.', {
|
||||
legacyCode: 'code_mismatch',
|
||||
fields: { factor: 'otp' },
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.#requireAccountTwoFactor(user, code);
|
||||
}
|
||||
|
||||
await this.stores.user.update(user.id, {
|
||||
otp_enabled: false,
|
||||
@@ -4479,11 +4505,14 @@ export class AuthController extends PuterController {
|
||||
}
|
||||
|
||||
const token = signStepUpToken(this.services.token, user as never);
|
||||
res.cookie(
|
||||
STEP_UP_COOKIE_NAME,
|
||||
token,
|
||||
stepUpCookieOptions(this.config),
|
||||
);
|
||||
const isApiHost = req.subdomains?.at(-1) === 'api';
|
||||
if (!isApiHost) {
|
||||
res.cookie(
|
||||
STEP_UP_COOKIE_NAME,
|
||||
token,
|
||||
stepUpCookieOptions(this.config),
|
||||
);
|
||||
}
|
||||
|
||||
// A browser reads its elevation back from the httpOnly cookie and never
|
||||
// needs the raw value; handing it to page JS would put the second factor
|
||||
@@ -4492,7 +4521,9 @@ export class AuthController extends PuterController {
|
||||
// password/TOTP — this only avoids needless exposure, it isn't a gate.
|
||||
const cookieName = this.config.cookie_name ?? 'puter_token';
|
||||
const usedSessionCookie =
|
||||
!!req.token && req.token === req.cookies?.[cookieName];
|
||||
!isApiHost &&
|
||||
!!req.token &&
|
||||
req.token === req.cookies?.[cookieName];
|
||||
res.json(
|
||||
usedSessionCookie ? { elevated: true } : { elevated: true, token },
|
||||
);
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { hashRecoveryCode } from '../../services/auth/OTPUtil.js';
|
||||
import {
|
||||
createTestUser,
|
||||
setupPuterTestEnv,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
|
||||
describe('2FA after signing in with a recovery code', () => {
|
||||
let env: PuterTestEnv;
|
||||
const password = 'lost-authenticator-password';
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = (
|
||||
path: string,
|
||||
body: Record<string, unknown>,
|
||||
cookie?: string,
|
||||
) =>
|
||||
fetch(new URL(path, env.origin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(cookie ? { cookie } : {}),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
it('allows an unused recovery code to disable 2FA after password revalidation', async () => {
|
||||
const username = `lost2fa_${uuidv4().slice(0, 8)}`;
|
||||
await createTestUser(env.server, { username, password });
|
||||
const user = (await env.server.stores.user.getByUsername(username))!;
|
||||
await env.server.stores.user.update(user.id, {
|
||||
email: `${username}@example.com`,
|
||||
email_confirmed: true,
|
||||
otp_enabled: true,
|
||||
otp_secret: 'TESTSECRETBASE32',
|
||||
otp_recovery_codes: ['RECOVER1', 'RECOVER2', 'RECOVER3']
|
||||
.map(hashRecoveryCode)
|
||||
.join(','),
|
||||
});
|
||||
|
||||
// Sign in with a recovery code: works, and yields a web session.
|
||||
const login = await post('/login', { username, password });
|
||||
expect(login.status).toBe(202);
|
||||
const { otp_jwt_token } = await login.json();
|
||||
const recovered = await post('/login/recovery-code', {
|
||||
token: otp_jwt_token,
|
||||
code: 'RECOVER1',
|
||||
});
|
||||
expect((await recovered.json()).next_step).toBe('complete');
|
||||
const cookie = recovered.headers
|
||||
.getSetCookie()
|
||||
.find((value) => value.startsWith('puter_auth_token='))!
|
||||
.split(';')[0]!;
|
||||
|
||||
const reused = await post(
|
||||
'/user-protected/disable-2fa',
|
||||
{ password, code: 'RECOVER1' },
|
||||
cookie,
|
||||
);
|
||||
expect(reused.status).toBe(401);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, { force: true }))!
|
||||
.otp_enabled,
|
||||
).toBeTruthy();
|
||||
|
||||
const disable = await post(
|
||||
'/user-protected/disable-2fa',
|
||||
{ password, code: 'RECOVER2' },
|
||||
cookie,
|
||||
);
|
||||
expect(disable.status).toBe(200);
|
||||
expect(await disable.json()).toMatchObject({ success: true });
|
||||
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.otp_enabled).toBeFalsy();
|
||||
expect(after!.otp_secret).toBeNull();
|
||||
expect(after!.otp_recovery_codes).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import type { Request, RequestHandler, Response } from 'express';
|
||||
import { beforeEach, describe, expect, it } from 'vitest';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { Context, runWithContext } from '../../core/context.js';
|
||||
import { isHttpError } from '../../core/http/HttpError.js';
|
||||
import { configureRateLimit } from '../../core/http/middleware/rateLimit.js';
|
||||
import { DriverController } from './DriverController.js';
|
||||
@@ -278,6 +278,86 @@ describe('DriverController — concurrent acquire/release', () => {
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
// A driver that keeps working after the caller hangs up (async video
|
||||
// polling) must keep counting against the cap until it settles.
|
||||
it('holds the slot while the driver still works after a disconnect', async () => {
|
||||
let finish = () => {};
|
||||
const driver = {
|
||||
...makeSyntheticDriver(),
|
||||
ping: () =>
|
||||
new Promise<string>((resolve) => {
|
||||
finish = () => resolve('pong');
|
||||
}),
|
||||
};
|
||||
const handler = captureCallHandler(buildController(driver));
|
||||
const req = () =>
|
||||
makeReq({ interface: 'test-iface', method: 'ping' }, 'working');
|
||||
|
||||
const res1 = new StubRes();
|
||||
const first = callInScope(handler, req(), res1 as unknown as Response);
|
||||
await new Promise((r) => setImmediate(r));
|
||||
res1.emit('close');
|
||||
await new Promise((r) => setImmediate(r));
|
||||
|
||||
await expect(
|
||||
callInScope(handler, req(), new StubRes() as unknown as Response),
|
||||
).rejects.toSatisfy(
|
||||
(e) =>
|
||||
isHttpError(e) &&
|
||||
(e as { statusCode: number }).statusCode === 429,
|
||||
);
|
||||
|
||||
finish();
|
||||
await first;
|
||||
await new Promise((r) => setImmediate(r));
|
||||
const res2 = new StubRes();
|
||||
const second = callInScope(handler, req(), res2 as unknown as Response);
|
||||
await new Promise((r) => setImmediate(r));
|
||||
finish();
|
||||
await second;
|
||||
res2.emit('finish');
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
it('holds the slot for work a driver keeps past its response', async () => {
|
||||
let settleWork = () => {};
|
||||
const driver = {
|
||||
...makeSyntheticDriver(),
|
||||
ping: async () => {
|
||||
Context.get('keepDriverSlot')?.(
|
||||
new Promise<void>((resolve) => {
|
||||
settleWork = resolve;
|
||||
}),
|
||||
);
|
||||
return 'pong';
|
||||
},
|
||||
};
|
||||
const handler = captureCallHandler(buildController(driver));
|
||||
const req = () =>
|
||||
makeReq({ interface: 'test-iface', method: 'ping' }, 'kept');
|
||||
|
||||
const res1 = new StubRes();
|
||||
await callInScope(handler, req(), res1 as unknown as Response);
|
||||
res1.emit('finish');
|
||||
await new Promise((r) => setImmediate(r));
|
||||
|
||||
await expect(
|
||||
callInScope(handler, req(), new StubRes() as unknown as Response),
|
||||
).rejects.toSatisfy(
|
||||
(e) =>
|
||||
isHttpError(e) &&
|
||||
(e as { statusCode: number }).statusCode === 429,
|
||||
);
|
||||
|
||||
settleWork();
|
||||
await new Promise((r) => setImmediate(r));
|
||||
const res2 = new StubRes();
|
||||
await callInScope(handler, req(), res2 as unknown as Response);
|
||||
res2.emit('finish');
|
||||
settleWork();
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
it('does not attach release listeners when the driver declares no concurrent config', async () => {
|
||||
// With no spec there is nothing to release, so the gate must not
|
||||
// wire `finish`/`close` release listeners. The client-disconnect
|
||||
|
||||
@@ -50,6 +50,7 @@ import { PermissionUtil } from '../../services/permission/permissionUtil.js';
|
||||
import type { WithLifecycle } from '../../types';
|
||||
import { withSpan } from '../../util/span.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import { CallSlot } from './callSlot.js';
|
||||
|
||||
type DriverInstance = WithLifecycle & Record<string, unknown>;
|
||||
|
||||
@@ -259,6 +260,17 @@ export class DriverController extends PuterController {
|
||||
// -- Handlers ----------------------------------------------------
|
||||
|
||||
#handleCall = async (req: Request, res: Response): Promise<void> => {
|
||||
const slot = new CallSlot();
|
||||
const call = this.#call(req, res, slot);
|
||||
slot.keep(call);
|
||||
await call;
|
||||
};
|
||||
|
||||
#call = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
slot: CallSlot,
|
||||
): Promise<void> => {
|
||||
const {
|
||||
interface: ifaceName,
|
||||
method,
|
||||
@@ -395,10 +407,9 @@ export class DriverController extends PuterController {
|
||||
|
||||
// Per-method rate-limit and concurrent specs both live on the
|
||||
// driver's resolved meta. Rate-limit is single-shot; concurrent
|
||||
// acquires a slot that must be released when the response is done
|
||||
// — we hook `res.finish` / `res.close` for that so streamed
|
||||
// responses hold their slot until the stream drains, and aborted
|
||||
// requests still give the slot back.
|
||||
// acquires a slot held until the response is done and the call's
|
||||
// work has settled, so streamed responses hold it until the stream
|
||||
// drains and work that continues after a disconnect still counts.
|
||||
const rateLimitSpec = resolvePerMethod(driverMeta?.rateLimit, method);
|
||||
if (
|
||||
!(await checkDriverRateLimit(req, ifaceName, method, rateLimitSpec))
|
||||
@@ -430,18 +441,8 @@ export class DriverController extends PuterController {
|
||||
legacyCode: 'too_many_requests',
|
||||
});
|
||||
}
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
void handle.release();
|
||||
};
|
||||
// If the handler throws before responding, the express error
|
||||
// handler will eventually send a response — `finish` fires then,
|
||||
// so we still release. `close` covers client aborts.
|
||||
res.once('finish', release);
|
||||
res.once('close', release);
|
||||
if (res.destroyed || res.writableFinished) release();
|
||||
slot.attach(handle, res);
|
||||
Context.set('keepDriverSlot', (work) => slot.keep(work));
|
||||
}
|
||||
|
||||
// Stash the requested driver name in Context so multi-provider
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'node:events';
|
||||
import type { Response } from 'express';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { CallSlot } from './callSlot.js';
|
||||
|
||||
const handle = () => ({
|
||||
release: vi.fn(() => Promise.resolve()),
|
||||
renew: vi.fn(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const deferred = () => {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((r) => {
|
||||
resolve = r;
|
||||
});
|
||||
return { promise, resolve };
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('CallSlot', () => {
|
||||
it('renews a slot held by a long call and stops once it is released', async () => {
|
||||
vi.useFakeTimers();
|
||||
const slot = new CallSlot();
|
||||
const slotHandle = handle();
|
||||
const res = new EventEmitter();
|
||||
const call = deferred();
|
||||
slot.keep(call.promise);
|
||||
slot.attach(slotHandle, res as unknown as Response);
|
||||
|
||||
res.emit('close');
|
||||
await vi.advanceTimersByTimeAsync(25 * 60 * 1000);
|
||||
expect(slotHandle.renew).toHaveBeenCalledTimes(2);
|
||||
expect(slotHandle.release).not.toHaveBeenCalled();
|
||||
|
||||
call.resolve();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(slotHandle.release).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(25 * 60 * 1000);
|
||||
expect(slotHandle.renew).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('releases once the response is done if no work is outstanding', async () => {
|
||||
const slot = new CallSlot();
|
||||
const slotHandle = handle();
|
||||
const res = new EventEmitter();
|
||||
slot.attach(slotHandle, res as unknown as Response);
|
||||
res.emit('finish');
|
||||
expect(slotHandle.release).toHaveBeenCalledOnce();
|
||||
expect(slotHandle.renew).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { Response } from 'express';
|
||||
|
||||
// Under the concurrency store's one-hour orphan TTL.
|
||||
const SLOT_RENEW_MS = 10 * 60 * 1000;
|
||||
|
||||
interface SlotHandle {
|
||||
release(): Promise<void>;
|
||||
renew?(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A driver call's concurrency slot, held until the response is done and the
|
||||
* call's work has settled, including work a driver carries on after the caller
|
||||
* hung up or was answered.
|
||||
*/
|
||||
export class CallSlot {
|
||||
#handle?: SlotHandle;
|
||||
#work = new Set<Promise<unknown>>();
|
||||
#responded = false;
|
||||
#released = false;
|
||||
#renew?: NodeJS.Timeout;
|
||||
|
||||
attach(handle: SlotHandle, res: Response): void {
|
||||
this.#handle = handle;
|
||||
const responded = () => {
|
||||
this.#responded = true;
|
||||
this.#maybeRelease();
|
||||
};
|
||||
res.once('finish', responded);
|
||||
res.once('close', responded);
|
||||
if (res.destroyed || res.writableFinished) responded();
|
||||
if (!this.#released && this.#work.size > 0) this.#startRenewing();
|
||||
}
|
||||
|
||||
keep(work: Promise<unknown>): void {
|
||||
if (this.#released) return;
|
||||
this.#work.add(work);
|
||||
if (this.#handle) this.#startRenewing();
|
||||
const settled = () => {
|
||||
this.#work.delete(work);
|
||||
this.#maybeRelease();
|
||||
};
|
||||
work.then(settled, settled);
|
||||
}
|
||||
|
||||
#startRenewing(): void {
|
||||
if (this.#renew) return;
|
||||
this.#renew = setInterval(() => {
|
||||
void this.#handle?.renew?.();
|
||||
}, SLOT_RENEW_MS);
|
||||
this.#renew.unref?.();
|
||||
}
|
||||
|
||||
#maybeRelease(): void {
|
||||
if (this.#released || !this.#handle) return;
|
||||
if (!this.#responded || this.#work.size > 0) return;
|
||||
this.#released = true;
|
||||
clearInterval(this.#renew);
|
||||
void this.#handle.release();
|
||||
}
|
||||
}
|
||||
@@ -139,14 +139,14 @@ describe('the delivery budgets', () => {
|
||||
});
|
||||
|
||||
it('pins the share-handle quotas by plan', () => {
|
||||
expect(EVENTS_KV_HANDLES_PER_USER.limit).toBe(512);
|
||||
expect(EVENTS_KV_HANDLES_PER_APP.limit).toBe(512);
|
||||
expect(EVENTS_KV_HANDLES_PER_USER.limit).toBe(25_000);
|
||||
expect(EVENTS_KV_HANDLES_PER_APP.limit).toBe(10_000);
|
||||
expect(
|
||||
limitFor(EVENTS_KV_HANDLES_PER_USER, DEFAULT_FREE_SUBSCRIPTION),
|
||||
).toBe(200);
|
||||
).toBe(5_000);
|
||||
expect(
|
||||
limitFor(EVENTS_KV_HANDLES_PER_APP, DEFAULT_FREE_SUBSCRIPTION),
|
||||
).toBe(128);
|
||||
).toBe(2_000);
|
||||
});
|
||||
|
||||
it('keeps `single` well under broadcast — each one costs far more', () => {
|
||||
|
||||
@@ -155,6 +155,13 @@ export const EVENTS_LIST_LIMIT = userWindow('events:list', 120);
|
||||
*/
|
||||
export const EVENTS_ACK_LIMIT = userWindow('events:ack', 600);
|
||||
|
||||
/**
|
||||
* How long a connected client holds a queued delivery before the events worker
|
||||
* may take it. A client that has no handler for it, or whose handler fails,
|
||||
* declines it at once; this only bounds one that never answers.
|
||||
*/
|
||||
export const EVENTS_SOCKET_LEASE_MS = 15_000;
|
||||
|
||||
/**
|
||||
* Share-handle mint + revoke calls per minute, per user.
|
||||
*
|
||||
@@ -176,20 +183,37 @@ export const KV_SHARE_NOTICE_LIMIT = userWindow(
|
||||
/**
|
||||
* Live share handles one account may hold out at a time, across every app.
|
||||
*
|
||||
* Each is a standing grant on part of the account's data, and revoking marks
|
||||
* rather than deletes, so without a ceiling the rate limit alone lets the rows
|
||||
* grow forever. A temporary account holds none: the grant outlives the session
|
||||
* that made it, and no account is left to take it back.
|
||||
* A backstop above the per-app cap: every count reads a covering index, so the
|
||||
* bound is on standing grants, not query cost. A temporary account holds none:
|
||||
* the grant outlives the session that made it, and no account is left to take
|
||||
* it back.
|
||||
*/
|
||||
export const EVENTS_KV_HANDLES_PER_USER = tiered(512, 200, 0);
|
||||
export const EVENTS_KV_HANDLES_PER_USER = tiered(25_000, 5_000, 0);
|
||||
|
||||
/**
|
||||
* Live share handles one app may hold out for one account.
|
||||
*
|
||||
* Separately bounded from the account cap so one namespace cannot consume the
|
||||
* whole budget on free accounts.
|
||||
* Live share handles one app may hold out for one account, so one namespace
|
||||
* cannot consume the whole account budget.
|
||||
*/
|
||||
export const EVENTS_KV_HANDLES_PER_APP = tiered(512, 128, 0);
|
||||
export const EVENTS_KV_HANDLES_PER_APP = tiered(10_000, 2_000, 0);
|
||||
|
||||
/**
|
||||
* Live share handles one owner may hold out to one holder, on every plan.
|
||||
* Bounds the rows a revoke or unshare walks to find the narrower handles inside
|
||||
* one.
|
||||
*/
|
||||
export const EVENTS_KV_HANDLES_PER_HOLDER = 256;
|
||||
|
||||
/**
|
||||
* Recipients one account may share with, per minute and per hour. A batch
|
||||
* spends one per recipient, on top of the per-call mint budget.
|
||||
*/
|
||||
export const KV_SHARE_GRANT_LIMITS = [
|
||||
userWindow('kvShare:grants:minute', 300),
|
||||
userWindow('kvShare:grants:hour', 2_000, 60 * 60_000),
|
||||
] as const;
|
||||
|
||||
/** Recipients one `POST /kv/shares` call may name. */
|
||||
export const KV_SHARE_BATCH_MAX = 50;
|
||||
|
||||
// -- Handler surface -------------------------------------------------
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
describe('recursive readdir with offset', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = async (path: string, token: string, body: unknown) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
it('skips entries before applying the recursive page limit', async () => {
|
||||
const { username, token } = env.users.user;
|
||||
const base = `/${username}/Documents/rec-offset`;
|
||||
for (const path of [base, `${base}/a`, `${base}/b`, `${base}/c`]) {
|
||||
expect((await post('/fs/mkdir', token, { path })).status).toBe(200);
|
||||
}
|
||||
|
||||
const list = async (body: Record<string, unknown>) => {
|
||||
const response = await post('/fs/readdir', token, {
|
||||
path: base,
|
||||
recursive: true,
|
||||
...body,
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
const page = (await response.json()) as {
|
||||
items: Array<{ name: string }>;
|
||||
};
|
||||
return page.items.map((item) => item.name);
|
||||
};
|
||||
|
||||
expect(await list({ limit: 3 })).toEqual(['a', 'b', 'c']);
|
||||
// Non-recursive honours offset.
|
||||
const flat = await post('/fs/readdir', token, {
|
||||
path: base,
|
||||
limit: 1,
|
||||
offset: 2,
|
||||
});
|
||||
expect(
|
||||
((await flat.json()) as Array<{ name: string }>).map((e) => e.name),
|
||||
).toEqual(['c']);
|
||||
expect(await list({ limit: 1, offset: 2 })).toEqual(['c']);
|
||||
expect(await list({ offset: 2 })).toEqual(['c']);
|
||||
expect(await list({ offset: 20 })).toEqual([]);
|
||||
expect(await list({ limit: 1, offset: 1, sortOrder: 'desc' })).toEqual([
|
||||
'b',
|
||||
]);
|
||||
const first = await post('/fs/readdir', token, {
|
||||
path: base,
|
||||
recursive: true,
|
||||
limit: 1,
|
||||
offset: 1,
|
||||
});
|
||||
const page = await first.json();
|
||||
expect(page.items.map((item: { name: string }) => item.name)).toEqual([
|
||||
'b',
|
||||
]);
|
||||
expect(page.cursor).toBeTypeOf('string');
|
||||
expect(await list({ limit: 1, cursor: page.cursor })).toEqual(['c']);
|
||||
// A cursor wins over offset, and an invalid offset is ignored, as
|
||||
// recursive listings did before they honoured offset.
|
||||
expect(
|
||||
await list({ limit: 1, cursor: page.cursor, offset: 1 }),
|
||||
).toEqual(['c']);
|
||||
expect(await list({ limit: 1, offset: -1 })).toEqual(['a']);
|
||||
});
|
||||
});
|
||||
@@ -1335,6 +1335,7 @@ export class FSController extends PuterController {
|
||||
parent.path,
|
||||
{
|
||||
limit,
|
||||
offset,
|
||||
cursor:
|
||||
typeof body.cursor === 'string'
|
||||
? body.cursor
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { makeActor } from '../../core/actor.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
describe('/auth/check-app-acl', () => {
|
||||
let env: PuterTestEnv;
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = (path: string, body: unknown, token: string) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const makeDirAndApp = async () => {
|
||||
const { username, token } = env.users.user;
|
||||
const server = env.server;
|
||||
const user = (await server.stores.user.getByUsername(username))!;
|
||||
const path = `/${username}/Documents/cacl-${uuidv4()}`;
|
||||
expect((await post('/fs/mkdir', { path }, token)).status).toBe(200);
|
||||
const entry = (await server.stores.fsEntry.getEntryByPath(path))!;
|
||||
|
||||
const app = await (
|
||||
server.stores.app.create as unknown as (
|
||||
f: Record<string, unknown>,
|
||||
o: { ownerUserId: number },
|
||||
) => Promise<{ uid: string; id: number }>
|
||||
)(
|
||||
{
|
||||
name: `cacl-${uuidv4()}`,
|
||||
title: 'Check app acl',
|
||||
index_url: `https://cacl-${uuidv4()}.test/`,
|
||||
},
|
||||
{ ownerUserId: user.id },
|
||||
);
|
||||
return { user, path, entry, app };
|
||||
};
|
||||
|
||||
it('allows a granted file without poisoning the app permission cache', async () => {
|
||||
const { token } = env.users.user;
|
||||
const server = env.server;
|
||||
const { user, path, entry, app } = await makeDirAndApp();
|
||||
|
||||
const grant = await post(
|
||||
'/auth/grant-user-app',
|
||||
{ app_uid: app.uid, permission: `fs:${entry.uuid}:read` },
|
||||
token,
|
||||
);
|
||||
expect(grant.status).toBe(200);
|
||||
|
||||
const res = await post(
|
||||
'/auth/check-app-acl',
|
||||
{ subject: { path }, app: app.uid, mode: 'read' },
|
||||
token,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(await res.json()).toEqual({ allowed: true });
|
||||
|
||||
const asApp = makeActor({ user, app: { uid: app.uid, id: app.id } });
|
||||
const descriptor = () => server.services.acl.fsDescriptor(path);
|
||||
expect(
|
||||
await server.services.acl.check(asApp, descriptor(), 'read'),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('denies an app with no grant', async () => {
|
||||
const { path, app } = await makeDirAndApp();
|
||||
const res = await post(
|
||||
'/auth/check-app-acl',
|
||||
{ subject: { path }, app: app.uid, mode: 'read' },
|
||||
env.users.user.token,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(await res.json()).toEqual({ allowed: false });
|
||||
});
|
||||
|
||||
it('refuses app and access tokens', async () => {
|
||||
const { token, apiToken } = env.users.user;
|
||||
const { path, entry, app } = await makeDirAndApp();
|
||||
const grant = await post(
|
||||
'/auth/grant-user-app',
|
||||
{ app_uid: app.uid, permission: `fs:${entry.uuid}:read` },
|
||||
token,
|
||||
);
|
||||
expect(grant.status).toBe(200);
|
||||
|
||||
const minted = await post(
|
||||
'/auth/get-user-app-token',
|
||||
{ app_uid: app.uid },
|
||||
token,
|
||||
);
|
||||
expect(minted.status).toBe(200);
|
||||
const appToken = ((await minted.json()) as { token: string }).token;
|
||||
|
||||
for (const credential of [appToken, apiToken]) {
|
||||
const res = await post(
|
||||
'/auth/check-app-acl',
|
||||
{ subject: { path }, app: app.uid, mode: 'read' },
|
||||
credential,
|
||||
);
|
||||
expect(res.status).toBe(403);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const post = (path: string, body: unknown) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${env.users.user.token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const write = (path: string, content: string) =>
|
||||
post('/fs/write', {
|
||||
fileMetadata: { path, size: content.length, contentType: 'text/plain' },
|
||||
fileContent: content,
|
||||
});
|
||||
|
||||
const read = async (path: string) => {
|
||||
const url = new URL('/read', env.apiOrigin);
|
||||
url.searchParams.set('file', path);
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${env.users.user.token}` },
|
||||
});
|
||||
return response.text();
|
||||
};
|
||||
|
||||
const makeDir = async (name: string) => {
|
||||
const path = `/${env.users.user.username}/Documents/${name}-${randomUUID().slice(0, 8)}`;
|
||||
expect((await post('/fs/mkdir', { path })).status).toBe(200);
|
||||
return path;
|
||||
};
|
||||
|
||||
describe('legacy /copy onto an existing file', () => {
|
||||
it('replaces the file with overwrite', async () => {
|
||||
const dir = await makeDir('copy-onto');
|
||||
expect((await write(`${dir}/a.txt`, 'aaa')).status).toBe(200);
|
||||
expect((await write(`${dir}/b.txt`, 'bbbbbb')).status).toBe(200);
|
||||
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination: `${dir}/b.txt`,
|
||||
overwrite: true,
|
||||
});
|
||||
expect(copy.status).toBe(200);
|
||||
const [result] = (await copy.json()) as Array<{
|
||||
copied: { name: string; path: string };
|
||||
overwritten?: { name: string };
|
||||
}>;
|
||||
expect(result!.copied.name).toBe('b.txt');
|
||||
expect(result!.overwritten?.name).toBe('b.txt');
|
||||
expect(await read(`${dir}/b.txt`)).toBe('aaa');
|
||||
expect(await read(`${dir}/a.txt`)).toBe('aaa');
|
||||
});
|
||||
|
||||
it('refuses without overwrite and leaves the file alone', async () => {
|
||||
const dir = await makeDir('copy-onto-refused');
|
||||
await write(`${dir}/a.txt`, 'aaa');
|
||||
await write(`${dir}/b.txt`, 'bbbbbb');
|
||||
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination: `${dir}/b.txt`,
|
||||
});
|
||||
expect(copy.status).toBe(409);
|
||||
expect(((await copy.json()) as { code: string }).code).toBe(
|
||||
'item_with_same_name_exists',
|
||||
);
|
||||
expect(await read(`${dir}/b.txt`)).toBe('bbbbbb');
|
||||
});
|
||||
|
||||
it('still copies into a directory destination under the same name', async () => {
|
||||
const dir = await makeDir('copy-into');
|
||||
await write(`${dir}/a.txt`, 'aaa');
|
||||
expect((await post('/fs/mkdir', { path: `${dir}/sub` })).status).toBe(
|
||||
200,
|
||||
);
|
||||
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination: `${dir}/sub`,
|
||||
});
|
||||
expect(copy.status).toBe(200);
|
||||
expect(await read(`${dir}/sub/a.txt`)).toBe('aaa');
|
||||
});
|
||||
|
||||
it('refuses to overwrite a file with itself', async () => {
|
||||
const dir = await makeDir('copy-self');
|
||||
await write(`${dir}/a.txt`, 'aaa');
|
||||
|
||||
for (const destination of [`${dir}/a.txt`, dir]) {
|
||||
const copy = await post('/copy', {
|
||||
source: `${dir}/a.txt`,
|
||||
destination,
|
||||
overwrite: true,
|
||||
});
|
||||
expect(copy.status).toBe(400);
|
||||
}
|
||||
expect(await read(`${dir}/a.txt`)).toBe('aaa');
|
||||
});
|
||||
});
|
||||
|
||||
describe('signed folder listing', () => {
|
||||
const signedListing = async (path: string) => {
|
||||
const signed = await post('/sign', {
|
||||
items: [{ path, action: 'read' }],
|
||||
});
|
||||
expect(signed.status).toBe(200);
|
||||
const readUrl = new URL(
|
||||
(await signed.json()).signatures[0].read_url as string,
|
||||
);
|
||||
return fetch(
|
||||
new URL(`${readUrl.pathname}${readUrl.search}`, env.apiOrigin),
|
||||
);
|
||||
};
|
||||
|
||||
it('lists every child of a folder larger than one query', async () => {
|
||||
const dir = await makeDir('big');
|
||||
const parent = (await env.server.stores.fsEntry.getEntryByPath(dir))!;
|
||||
const total = 10_050;
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
for (let start = 0; start < total; start += 500) {
|
||||
const rows: unknown[][] = [];
|
||||
for (let i = start; i < Math.min(total, start + 500); i++) {
|
||||
const name = `f${String(i).padStart(5, '0')}.txt`;
|
||||
rows.push([
|
||||
randomUUID(),
|
||||
parent.userId,
|
||||
parent.id,
|
||||
parent.uuid,
|
||||
name,
|
||||
`${dir}/${name}`,
|
||||
0,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
0,
|
||||
]);
|
||||
}
|
||||
await env.server.clients.db.write(
|
||||
`INSERT INTO fsentries (uuid, user_id, parent_id, parent_uid, name, path, is_dir, modified, created, accessed, size) VALUES ${rows.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
|
||||
rows.flat(),
|
||||
);
|
||||
}
|
||||
|
||||
const response = await signedListing(dir);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toMatch(
|
||||
/^application\/json/,
|
||||
);
|
||||
const children = (await response.json()) as Array<{
|
||||
fsentry_name: string;
|
||||
write_url?: string;
|
||||
}>;
|
||||
expect(children).toHaveLength(total);
|
||||
expect(children[0]!.fsentry_name).toBe('f00000.txt');
|
||||
expect(children.at(-1)!.fsentry_name).toBe('f10049.txt');
|
||||
expect(new Set(children.map((c) => c.fsentry_name)).size).toBe(total);
|
||||
expect(children.some((c) => c.write_url)).toBe(false);
|
||||
}, 120_000);
|
||||
|
||||
it('answers a small folder in one body', async () => {
|
||||
const dir = await makeDir('small');
|
||||
await write(`${dir}/one.txt`, '1');
|
||||
const response = await signedListing(dir);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-length')).not.toBeNull();
|
||||
const children = (await response.json()) as Array<{
|
||||
fsentry_name: string;
|
||||
}>;
|
||||
expect(children.map((c) => c.fsentry_name)).toEqual(['one.txt']);
|
||||
});
|
||||
});
|
||||
@@ -153,6 +153,34 @@ describe('legacy file downloads over HTTP', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(routes)(
|
||||
'GET %s answers a ranged read of an empty file with an empty 200',
|
||||
async (route) => {
|
||||
const file = await writeFile(
|
||||
`empty-${route.replace(/\W/g, '')}.bin`,
|
||||
Buffer.alloc(0),
|
||||
);
|
||||
const ranged = await requestFor(route, file, {
|
||||
headers: { Range: 'bytes=0-511' },
|
||||
});
|
||||
expect(ranged.status).toBe(200);
|
||||
expect(await ranged.text()).toBe('');
|
||||
},
|
||||
);
|
||||
|
||||
it('GET /read answers byte_count on an empty file with an empty 200', async () => {
|
||||
const file = await writeFile('empty-byte-count.bin', Buffer.alloc(0));
|
||||
for (const query of ['byte_count=512', 'offset=0']) {
|
||||
const url = new URL(`/read?${query}`, env.apiOrigin);
|
||||
url.searchParams.set('file', file.path);
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${env.users.user.token}` },
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.text()).toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
// Storage hands over one chunk and then goes quiet, as a large object
|
||||
// does while the client is still reading.
|
||||
const stallReadsOf = (objectKey: string) => {
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import type { Request, RequestHandler, Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { PuterRouter } from '../../core/http/PuterRouter.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { LegacyFSController } from './LegacyFSController.js';
|
||||
|
||||
const PNG_2X2 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEUlEQVR4nGP4z8DwH4QZYAwAR8oH+WdZbrcAAAAASUVORK5CYII=';
|
||||
|
||||
let server: PuterServer;
|
||||
let handler: RequestHandler;
|
||||
type Driver = {
|
||||
create: (a: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer({
|
||||
api_base_url: 'http://api.test.local',
|
||||
static_hosting_domain: 'site.test',
|
||||
} as unknown as Parameters<typeof setupTestServer>[0]);
|
||||
await server.services.appIcon.ensureIconsDirectory();
|
||||
const controller = server.controllers
|
||||
.legacyFs as unknown as LegacyFSController;
|
||||
const router = new PuterRouter();
|
||||
controller.registerRoutes(router);
|
||||
handler = router.routes.find(
|
||||
(r) =>
|
||||
r.path === '/get-launch-apps' &&
|
||||
String(r.method).toLowerCase() === 'get',
|
||||
)!.handler as RequestHandler;
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
const makeActor = async (): Promise<{ actor: Actor; id: number }> => {
|
||||
const username = `v13-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const u = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
const r = (await server.stores.user.getById(u.id))!;
|
||||
return {
|
||||
id: r.id,
|
||||
actor: {
|
||||
user: {
|
||||
id: r.id,
|
||||
uuid: r.uuid,
|
||||
username: r.username,
|
||||
email: r.email ?? null,
|
||||
email_confirmed: true,
|
||||
},
|
||||
} as unknown as Actor,
|
||||
};
|
||||
};
|
||||
|
||||
const createViaDriver = async (actor: Actor, icon: string) =>
|
||||
runWithContext({ actor }, () =>
|
||||
(server.drivers.apps as unknown as Driver).create({
|
||||
object: {
|
||||
name: `v13-${Math.random().toString(36).slice(2, 10)}`,
|
||||
title: 'V',
|
||||
index_url: `https://v13-${Math.random().toString(36).slice(2, 10)}.test/`,
|
||||
icon,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const launchApps = async (actor: Actor, appUid: string, userId: number) => {
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `app_opens` (`app_uid`, `user_id`, `ts`) VALUES (?, ?, ?)',
|
||||
[appUid, userId, Math.floor(Date.now() / 1000)],
|
||||
);
|
||||
let body: { recent: Array<Record<string, unknown>> } | undefined;
|
||||
const res = {
|
||||
json: (v: unknown) => {
|
||||
body = v as typeof body;
|
||||
return res;
|
||||
},
|
||||
status: () => res,
|
||||
setHeader: () => res,
|
||||
} as unknown as Response;
|
||||
await handler(
|
||||
{
|
||||
query: { icon_size: '64' },
|
||||
headers: {},
|
||||
body: {},
|
||||
actor,
|
||||
} as unknown as Request,
|
||||
res,
|
||||
() => {},
|
||||
);
|
||||
return body!.recent.find((e) => e.uuid === appUid)!;
|
||||
};
|
||||
|
||||
describe('/get-launch-apps recent icon, verified', () => {
|
||||
it('a valid data-URL icon written through the driver is migrated to a short URL', async () => {
|
||||
const { actor, id } = await makeActor();
|
||||
const app = await createViaDriver(
|
||||
actor,
|
||||
`data:image/png;base64,${PNG_2X2}`,
|
||||
);
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
const row = await server.stores.app.getByUid(app.uid as string);
|
||||
expect(String(row?.icon).slice(0, 80)).toMatch(
|
||||
/^https?:\/\/.*\/app-icon\/app-/,
|
||||
);
|
||||
},
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
const entry = await launchApps(actor, app.uid as string, id);
|
||||
expect(String(entry.icon).length).toBeLessThan(200);
|
||||
expect(entry.iconCdnUrl).toMatch(/-64\.png$/);
|
||||
}, 30_000);
|
||||
|
||||
it('returns an icon endpoint when migration cannot decode the stored icon', async () => {
|
||||
const { actor, id } = await makeActor();
|
||||
// PNG signature passes the write-path sniff; sharp rejects the rest.
|
||||
const corrupt = Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
Buffer.alloc(1024 * 1024, 0x41),
|
||||
]).toString('base64');
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
let app: Record<string, unknown>;
|
||||
try {
|
||||
app = await createViaDriver(
|
||||
actor,
|
||||
`data:image/png;base64,${corrupt}`,
|
||||
);
|
||||
// Give the background pipeline time to try and fail.
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(
|
||||
warn.mock.calls.some(
|
||||
([m]) => m === '[app-icon] icon processing failed',
|
||||
),
|
||||
).toBe(true),
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
const row = await server.stores.app.getByUid(app!.uid as string);
|
||||
expect(String(row?.icon).startsWith('data:image/png')).toBe(true);
|
||||
const entry = await launchApps(actor, app!.uid as string, id);
|
||||
expect(entry.icon).toBe(
|
||||
`http://api.test.local/app-icon/${app!.uid}/64`,
|
||||
);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -5293,4 +5293,39 @@ describe('LegacyFSController GET /get-launch-apps', () => {
|
||||
|
||||
expect(returned).toEqual(recentUids);
|
||||
});
|
||||
|
||||
it('skips the recent opens for recent=0 and keeps recommended', async () => {
|
||||
const { actor, userId } = await makeUser();
|
||||
const app = await makeHostedApp(userId, 'unregistered-skip');
|
||||
await recordOpen(userId, app.uid);
|
||||
|
||||
const call = async (query: Record<string, unknown>) => {
|
||||
const { res, captured } = makeRes();
|
||||
await withActor(actor, async () => {
|
||||
await getLaunchAppsHandler()(
|
||||
makeReq({ actor, query }),
|
||||
res,
|
||||
() => {
|
||||
throw new Error('handler called next() unexpectedly');
|
||||
},
|
||||
);
|
||||
});
|
||||
return captured.body as {
|
||||
recommended: unknown[];
|
||||
recent: unknown[];
|
||||
};
|
||||
};
|
||||
const full = await call({});
|
||||
expect(full.recent.length).toBeGreaterThan(0);
|
||||
|
||||
const opens = vi.spyOn(server.stores.app, 'getRecentAppOpens');
|
||||
try {
|
||||
const skipped = await call({ recent: '0' });
|
||||
expect(skipped.recent).toEqual([]);
|
||||
expect(skipped.recommended).toEqual(full.recommended);
|
||||
expect(opens).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
opens.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
assertResolvedActor,
|
||||
isAccessTokenActor,
|
||||
isAppActor,
|
||||
isPlainUserActor,
|
||||
makeActor,
|
||||
} from '../../core/actor.js';
|
||||
import { Context } from '../../core/context.js';
|
||||
@@ -95,6 +96,7 @@ import {
|
||||
assertAccess,
|
||||
assertCanCreate,
|
||||
assertCanMoveInto,
|
||||
contextBoundStream,
|
||||
emitGuiItemEvent,
|
||||
expandClientPath,
|
||||
findOverwriteTarget,
|
||||
@@ -102,6 +104,7 @@ import {
|
||||
getString,
|
||||
type GuiItemEventName,
|
||||
loadLegacyAssociatedApps,
|
||||
rangeForEntry,
|
||||
replacedByMove,
|
||||
resolveReadRange,
|
||||
resolveV1Selector,
|
||||
@@ -381,7 +384,9 @@ export class LegacyFSController extends PuterController {
|
||||
|
||||
let recent: unknown[] = [];
|
||||
const userId = req.actor?.user?.id;
|
||||
if (userId) {
|
||||
// `recent=0`: a caller that only shows `recommended`.
|
||||
const wantsRecent = req.query.recent !== '0';
|
||||
if (userId && wantsRecent) {
|
||||
const recentUids = await this.stores.app.getRecentAppOpens(
|
||||
userId,
|
||||
{ limit: 10 },
|
||||
@@ -418,9 +423,9 @@ export class LegacyFSController extends PuterController {
|
||||
return {
|
||||
...toAppSummary(app, {
|
||||
config: this.config,
|
||||
apiBaseUrl: this.config.api_base_url,
|
||||
iconSize,
|
||||
}),
|
||||
icon: app.icon ?? null,
|
||||
index_url: backingGone ? null : app.index_url,
|
||||
...(backingGone
|
||||
? { privateAccess: buildHostedBackingDenial() }
|
||||
@@ -801,10 +806,23 @@ export class LegacyFSController extends PuterController {
|
||||
this.stores.fsEntry,
|
||||
body.source,
|
||||
);
|
||||
const destinationParent = await resolveV1Selector(
|
||||
const destination = await resolveV1Selector(
|
||||
this.stores.fsEntry,
|
||||
body.destination,
|
||||
);
|
||||
// A file destination names the copy itself, replaced only with
|
||||
// `overwrite`.
|
||||
let newName = getString(body, 'new_name');
|
||||
let destinationParent = destination;
|
||||
if (!destination.isDir && !newName && destination.parentUid) {
|
||||
const parent = await this.stores.fsEntry.getEntryByUuid(
|
||||
destination.parentUid,
|
||||
);
|
||||
if (parent) {
|
||||
destinationParent = parent;
|
||||
newName = destination.name;
|
||||
}
|
||||
}
|
||||
|
||||
await assertAccess(
|
||||
this.services.acl,
|
||||
@@ -829,14 +847,14 @@ export class LegacyFSController extends PuterController {
|
||||
? await findOverwriteTarget(
|
||||
this.stores.fsEntry,
|
||||
destinationParent,
|
||||
getString(body, 'new_name') ?? source.name,
|
||||
newName ?? source.name,
|
||||
)
|
||||
: null;
|
||||
|
||||
const copy = await this.services.fs.copy(userId, {
|
||||
source,
|
||||
destinationParent,
|
||||
newName: getString(body, 'new_name'),
|
||||
newName,
|
||||
overwrite: overwriteRequested,
|
||||
dedupeName: getBoolean(body, 'dedupe_name', 'change_name') ?? false,
|
||||
});
|
||||
@@ -1289,7 +1307,10 @@ export class LegacyFSController extends PuterController {
|
||||
});
|
||||
}
|
||||
|
||||
const range = resolveReadRange(req.headers.range, query);
|
||||
const range = rangeForEntry(
|
||||
entry,
|
||||
resolveReadRange(req.headers.range, query),
|
||||
);
|
||||
const download = await this.services.fs.readContent(entry, {
|
||||
range,
|
||||
});
|
||||
@@ -1790,19 +1811,50 @@ export class LegacyFSController extends PuterController {
|
||||
// owner, so the layout it reveals is theirs to reveal, and the child
|
||||
// signatures expire; the paths are the only part that outlives them.
|
||||
if (entry.isDir) {
|
||||
const children = await this.services.fs.listDirectory(entry.uuid);
|
||||
const signedChildren = children.map((child) => {
|
||||
const signChild = (child: FSEntry) => {
|
||||
const { write_url: _, ...rest } = signEntry(child, signingCfg, {
|
||||
ttlSeconds: NON_OWNER_SIGNATURE_TTL_SECONDS,
|
||||
});
|
||||
return { ...rest, path: maskEntryPath(child) };
|
||||
});
|
||||
res.json(signedChildren);
|
||||
};
|
||||
const pages = this.services.fs.listDirectoryPages(entry.uuid);
|
||||
const first = await pages.next();
|
||||
const firstPage = first.done ? [] : first.value;
|
||||
const second = await pages.next();
|
||||
if (second.done) {
|
||||
res.json(firstPage.map(signChild));
|
||||
return;
|
||||
}
|
||||
|
||||
// Past one page the listing streams, so a large folder isn't held
|
||||
// in memory whole. A failure partway cuts the response off.
|
||||
const secondPage = second.value;
|
||||
const serialize = (child: FSEntry) =>
|
||||
JSON.stringify(signChild(child));
|
||||
async function* allPages() {
|
||||
yield firstPage;
|
||||
yield secondPage;
|
||||
yield* pages;
|
||||
}
|
||||
async function* body() {
|
||||
let separator = '[';
|
||||
for await (const page of allPages()) {
|
||||
if (page.length === 0) continue;
|
||||
yield separator + page.map(serialize).join(',');
|
||||
separator = ',';
|
||||
}
|
||||
yield separator === '[' ? '[]' : ']';
|
||||
}
|
||||
res.type('application/json');
|
||||
await pipeToResponse(contextBoundStream(body()), res);
|
||||
return;
|
||||
}
|
||||
|
||||
// File: stream bytes with Range support.
|
||||
const range = resolveReadRange(req.headers.range, query);
|
||||
const range = rangeForEntry(
|
||||
entry,
|
||||
resolveReadRange(req.headers.range, query),
|
||||
);
|
||||
const download = await this.services.fs.readContent(entry, {
|
||||
range,
|
||||
});
|
||||
@@ -1992,6 +2044,11 @@ export class LegacyFSController extends PuterController {
|
||||
*/
|
||||
checkAppAcl = async (req: Request, res: Response): Promise<void> => {
|
||||
const actor = this.#requireActor(req);
|
||||
// Sessions only: for an app, this would be a window onto its neighbours' grants.
|
||||
if (!isPlainUserActor(actor))
|
||||
throw new HttpError(403, 'actor must be a user', {
|
||||
legacyCode: 'forbidden',
|
||||
});
|
||||
const body = asRecord(req.body);
|
||||
|
||||
const subjectRef = body.subject;
|
||||
@@ -2022,7 +2079,7 @@ export class LegacyFSController extends PuterController {
|
||||
legacyCode: 'subject_does_not_exist',
|
||||
});
|
||||
|
||||
let app: { uid: string } | null = null;
|
||||
let app: { uid: string; id: number } | null = null;
|
||||
if (typeof appRef === 'string') {
|
||||
app =
|
||||
(await this.stores.app.getByUid(appRef)) ??
|
||||
@@ -2036,7 +2093,7 @@ export class LegacyFSController extends PuterController {
|
||||
// Build an actor-under-user shape for the check.
|
||||
const actorForApp = makeActor({
|
||||
user: actor.user,
|
||||
app: { uid: (app as { uid: string }).uid },
|
||||
app: { uid: app.uid, id: app.id },
|
||||
});
|
||||
const allowed = await acl.check(actorForApp, descriptor, mode);
|
||||
res.json({ allowed });
|
||||
@@ -2084,10 +2141,12 @@ export class LegacyFSController extends PuterController {
|
||||
// by default, so opt in by naming the downloader as who pays.
|
||||
req.egressActor = actor;
|
||||
|
||||
const range =
|
||||
const range = rangeForEntry(
|
||||
entry,
|
||||
typeof req.headers.range === 'string'
|
||||
? req.headers.range
|
||||
: undefined;
|
||||
: undefined,
|
||||
);
|
||||
const download = await this.services.fs.readContent(entry, {
|
||||
range,
|
||||
});
|
||||
|
||||
@@ -17,15 +17,19 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { Writable } from 'node:stream';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { Context, runWithContext } from '../../core/context.js';
|
||||
import type { PuterServer } from '../../server.js';
|
||||
import { createTestUser, setupTestServer } from '../../testUtil.js';
|
||||
import {
|
||||
asRecord,
|
||||
contextBoundStream,
|
||||
fsEntryMimeType,
|
||||
getBoolean,
|
||||
getString,
|
||||
loadLegacyAssociatedApps,
|
||||
rangeForEntry,
|
||||
resolveReadRange,
|
||||
signEntry,
|
||||
signEntryThumbnail,
|
||||
@@ -705,3 +709,50 @@ describe('resolveReadRange', () => {
|
||||
).toThrow('Invalid byte_count');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rangeForEntry', () => {
|
||||
it('drops the range for an empty file only', () => {
|
||||
const entry = (size: number) => ({ size }) as FSEntry;
|
||||
expect(rangeForEntry(entry(0), 'bytes=0-511')).toBeUndefined();
|
||||
expect(rangeForEntry(entry(10), 'bytes=0-4')).toBe('bytes=0-4');
|
||||
});
|
||||
});
|
||||
|
||||
describe('contextBoundStream', () => {
|
||||
it("reads every chunk in the caller's context", async () => {
|
||||
// A sink slower than the source that completes writes from outside
|
||||
// any request, so later chunks are asked for from its drain.
|
||||
const pending: Array<() => void> = [];
|
||||
const flush = runWithContext({}, () =>
|
||||
setInterval(() => pending.splice(0).forEach((done) => done()), 5),
|
||||
);
|
||||
const sink = new Writable({
|
||||
highWaterMark: 1,
|
||||
write(_chunk, _encoding, done) {
|
||||
pending.push(done);
|
||||
},
|
||||
});
|
||||
const seen: unknown[] = [];
|
||||
try {
|
||||
await runWithContext({ actor: 'caller' as never }, async () => {
|
||||
async function* chunks() {
|
||||
for (let i = 0; i < 40; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
seen.push(Context.get('actor'));
|
||||
yield 'x';
|
||||
}
|
||||
}
|
||||
await new Promise((resolve, reject) => {
|
||||
contextBoundStream(chunks())
|
||||
.pipe(sink)
|
||||
.on('finish', resolve)
|
||||
.on('error', reject);
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
clearInterval(flush);
|
||||
}
|
||||
expect(seen).toHaveLength(40);
|
||||
expect(seen.every((actor) => actor === 'caller')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -17,7 +17,9 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { AsyncResource } from 'node:async_hooks';
|
||||
import { posix as pathPosix } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { contentType as contentTypeFromMime } from 'mime-types';
|
||||
import type { FSEntry } from '../../stores/fs/FSEntry.js';
|
||||
import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js';
|
||||
@@ -150,6 +152,29 @@ export function resolveReadRange(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A response body read from `chunks` for `pipeToResponse`. Each chunk is pulled
|
||||
* in the caller's async context, which share masking reads, not in whichever
|
||||
* socket event asked for more.
|
||||
*/
|
||||
export function contextBoundStream(
|
||||
chunks: AsyncGenerator<string, void, undefined>,
|
||||
): Readable {
|
||||
const next = AsyncResource.bind(() => chunks.next());
|
||||
const stop = AsyncResource.bind(() => chunks.return(undefined));
|
||||
return Readable.from({
|
||||
[Symbol.asyncIterator]: () => ({ next, return: stop }),
|
||||
});
|
||||
}
|
||||
|
||||
/** No range is satisfiable on an empty file, so it reads whole: empty, 200. */
|
||||
export function rangeForEntry(
|
||||
entry: FSEntry,
|
||||
range: string | undefined,
|
||||
): string | undefined {
|
||||
return entry.size === 0 ? undefined : range;
|
||||
}
|
||||
|
||||
// Accepts either `{ path }` or `{ uid }` or `{ id }` or `{ parent, name }`
|
||||
// from a legacy body field. Returns a resolved entry (throwing 404 if not
|
||||
// found or 400 if no usable ref is present).
|
||||
|
||||
@@ -17,6 +17,9 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type { Request, RequestHandler, Response } from 'express';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
@@ -466,3 +469,63 @@ describe('HomepageController GET /show/*splat', () => {
|
||||
expect(html).toContain('/alice/Documents');
|
||||
});
|
||||
});
|
||||
|
||||
// ── /sw.js ──────────────────────────────────────────────────────────
|
||||
|
||||
describe('HomepageController GET /sw.js', () => {
|
||||
const headersOf = (res: Response): Record<string, string> =>
|
||||
Object.fromEntries(
|
||||
(res.set as ReturnType<typeof vi.fn>).mock.calls.map(
|
||||
([key, value]) => [String(key).toLowerCase(), String(value)],
|
||||
),
|
||||
);
|
||||
|
||||
it('serves the worker with root scope, as JavaScript, revalidated each time', async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), 'puter-sw-route-'));
|
||||
await mkdir(path.join(root, 'dist'));
|
||||
await writeFile(path.join(root, 'dist', 'sw.js'), '// worker\n');
|
||||
const config = server.services.homepage.config as {
|
||||
gui_assets_root?: string;
|
||||
env?: string;
|
||||
};
|
||||
const prev = { root: config.gui_assets_root, env: config.env };
|
||||
config.gui_assets_root = root;
|
||||
config.env = 'prod';
|
||||
const { res, captured } = makeRes();
|
||||
try {
|
||||
await callRoute('get', '/sw.js', makeReq({ path: '/sw.js' }), res);
|
||||
} finally {
|
||||
config.gui_assets_root = prev.root;
|
||||
config.env = prev.env;
|
||||
}
|
||||
expect(captured.statusCode).toBe(200);
|
||||
expect(captured.body).toBe('// worker\n');
|
||||
expect(headersOf(res)).toMatchObject({
|
||||
'content-type': 'text/javascript; charset=utf-8',
|
||||
'cache-control': 'no-cache',
|
||||
'service-worker-allowed': '/',
|
||||
});
|
||||
});
|
||||
|
||||
it('is a 404 when there is no worker to serve', async () => {
|
||||
const config = server.services.homepage.config as {
|
||||
gui_assets_root?: string;
|
||||
};
|
||||
const prev = config.gui_assets_root;
|
||||
config.gui_assets_root = path.join(tmpdir(), 'no-such-gui-root');
|
||||
const { res, captured } = makeRes();
|
||||
try {
|
||||
await callRoute('get', '/sw.js', makeReq({ path: '/sw.js' }), res);
|
||||
} finally {
|
||||
config.gui_assets_root = prev;
|
||||
}
|
||||
expect(captured.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('is registered for the root subdomain only', () => {
|
||||
const route = router.routes.find(
|
||||
(r) => r.method === 'get' && r.path === '/sw.js',
|
||||
);
|
||||
expect(route?.options.subdomain).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -198,6 +198,20 @@ export class HomepageController extends PuterController {
|
||||
return sendShell(req, res, {}, launch);
|
||||
});
|
||||
|
||||
// -- /sw.js - the service worker, scoped to the whole origin --
|
||||
|
||||
router.get('/sw.js', { subdomain: '' }, async (_req, res) => {
|
||||
const source = await homepage.serviceWorkerSource();
|
||||
if (source === null) {
|
||||
res.status(404).send('Not found');
|
||||
return;
|
||||
}
|
||||
res.set('Content-Type', 'text/javascript; charset=utf-8');
|
||||
res.set('Cache-Control', 'no-cache');
|
||||
res.set('Service-Worker-Allowed', '/');
|
||||
res.send(source);
|
||||
});
|
||||
|
||||
// -- Fallback static mount -----------------------------------
|
||||
// Serves lingering GUI assets (images, fonts, lib files, etc.)
|
||||
// out of <gui_assets_root>/src. Falls through to the 404 handler
|
||||
|
||||
@@ -198,7 +198,7 @@ describe('HostingController /delete-site', () => {
|
||||
);
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
'subdomain.delete',
|
||||
{ subdomain: site.subdomain, uid: String(site.uuid) },
|
||||
{ subdomain: site.subdomain, uid: String(site.uuid), userId },
|
||||
{},
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -34,7 +34,15 @@ import {
|
||||
type TestUserCredentials,
|
||||
} from '../../testUtil.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
import { EVENTS_KV_HANDLE_LIMIT } from '../events/limits.js';
|
||||
import { claimRateLimit } from '../../core/http/middleware/rateLimit.js';
|
||||
import { seedKvShareHandles } from '../../services/kvShare/kvShareTestUtil.js';
|
||||
import { KV_GLOBAL_APP_KEY } from '../../stores/systemKv/SystemKVStore.js';
|
||||
import {
|
||||
EVENTS_KV_HANDLE_LIMIT,
|
||||
EVENTS_KV_HANDLES_PER_USER,
|
||||
KV_SHARE_BATCH_MAX,
|
||||
KV_SHARE_GRANT_LIMITS,
|
||||
} from '../events/limits.js';
|
||||
|
||||
const BOOT_TIMEOUT_MS = 120_000;
|
||||
|
||||
@@ -57,9 +65,11 @@ const call = (
|
||||
const resetBudgets = async (env: PuterTestEnv) => {
|
||||
for (const name of [env.users.user.username, env.users.other.username]) {
|
||||
const user = await env.server.stores.user.getByUsername(name);
|
||||
await env.server.clients.redis.del(
|
||||
`rate:${EVENTS_KV_HANDLE_LIMIT.scope}:${user!.id}`,
|
||||
);
|
||||
for (const scope of [
|
||||
EVENTS_KV_HANDLE_LIMIT.scope,
|
||||
...KV_SHARE_GRANT_LIMITS.map((window) => window.scope),
|
||||
])
|
||||
await env.server.clients.redis.del(`rate:${scope}:${user!.id}`);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -293,6 +303,325 @@ describe('/kv/shares', () => {
|
||||
const res = await req('GET', '/kv/shares');
|
||||
expect(res.status).toBeGreaterThanOrEqual(400);
|
||||
});
|
||||
|
||||
describe('filtering a listing', () => {
|
||||
type Page = { items: Array<Record<string, unknown>>; total?: number };
|
||||
let appUid: string;
|
||||
let inApp: Record<string, unknown>;
|
||||
let global: Record<string, unknown>;
|
||||
|
||||
const page = async (path: string, token: string) => {
|
||||
const res = await req('GET', path, token);
|
||||
return { status: res.status, body: (await res.json()) as Page };
|
||||
};
|
||||
const handlesOf = (body: Page) => body.items.map((row) => row.handle);
|
||||
|
||||
const makeApp = async () => {
|
||||
const owner = await env.server.stores.user.getByUsername(
|
||||
env.users.user.username,
|
||||
);
|
||||
const name = `kv-http-${uuidv4().slice(0, 8)}`;
|
||||
return env.server.stores.app.create(
|
||||
{
|
||||
name,
|
||||
title: 'Listing App',
|
||||
index_url: `https://${name}.example.test/index.html`,
|
||||
},
|
||||
{ ownerUserId: owner!.id },
|
||||
);
|
||||
};
|
||||
|
||||
/** The token the desktop launches the app with, for `token`'s user. */
|
||||
const appToken = async (token: string, uid: string) => {
|
||||
const res = await req('POST', '/auth/get-user-app-token', token, {
|
||||
app_uid: uid,
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
return ((await res.json()) as { token: string }).token;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
await resetBudgets(env);
|
||||
appUid = (await makeApp()).uid;
|
||||
inApp = await shareWithOther({ appUid });
|
||||
global = await shareWithOther();
|
||||
});
|
||||
|
||||
it('lists every app`s shares for the account, or one app`s, or the app-less store`s with `null`', async () => {
|
||||
for (const path of ['/kv/shares', '/kv/shares/with-me']) {
|
||||
const token =
|
||||
path === '/kv/shares'
|
||||
? env.users.user.token
|
||||
: env.users.other.token;
|
||||
const all = await page(`${path}?limit=200`, token);
|
||||
expect(handlesOf(all.body)).toEqual(
|
||||
expect.arrayContaining([inApp.handle, global.handle]),
|
||||
);
|
||||
|
||||
const one = await page(
|
||||
`${path}?limit=200&appUid=${appUid}`,
|
||||
token,
|
||||
);
|
||||
expect(handlesOf(one.body)).toEqual([inApp.handle]);
|
||||
|
||||
const appLess = await page(
|
||||
`${path}?limit=200&appUid=null`,
|
||||
token,
|
||||
);
|
||||
expect(handlesOf(appLess.body)).toContain(global.handle);
|
||||
expect(
|
||||
appLess.body.items.every((row) => row.appUid === null),
|
||||
).toBe(true);
|
||||
|
||||
const bad = await page(`${path}?appUid=not-an-app`, token);
|
||||
expect(bad.status).toBe(400);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps an app to its own namespace, refusing any other it names', async () => {
|
||||
const token = await appToken(env.users.other.token, appUid);
|
||||
const own = await page('/kv/shares/with-me?limit=200', token);
|
||||
expect(handlesOf(own.body)).toEqual([inApp.handle]);
|
||||
const named = await page(
|
||||
`/kv/shares/with-me?limit=200&appUid=${appUid}`,
|
||||
token,
|
||||
);
|
||||
expect(handlesOf(named.body)).toEqual([inApp.handle]);
|
||||
|
||||
const otherApp = (await makeApp()).uid;
|
||||
for (const appUidParam of ['null', otherApp]) {
|
||||
const refused = await page(
|
||||
`/kv/shares/with-me?appUid=${appUidParam}`,
|
||||
token,
|
||||
);
|
||||
expect(refused.status).toBe(403);
|
||||
expect(refused.body).toMatchObject({
|
||||
code: 'events_kv_handle_outside_namespace',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('narrows what was shared out to live or revoked shares, all by default', async () => {
|
||||
const gone = await shareWithOther({ appUid });
|
||||
await req(
|
||||
'DELETE',
|
||||
`/kv/shares/${gone.handle}`,
|
||||
env.users.user.token,
|
||||
);
|
||||
const at = async (state?: string) =>
|
||||
page(
|
||||
`/kv/shares?limit=200&appUid=${appUid}` +
|
||||
(state ? `&state=${state}` : ''),
|
||||
env.users.user.token,
|
||||
);
|
||||
|
||||
expect(handlesOf((await at()).body)).toEqual([
|
||||
inApp.handle,
|
||||
gone.handle,
|
||||
]);
|
||||
expect(handlesOf((await at('all')).body)).toEqual([
|
||||
inApp.handle,
|
||||
gone.handle,
|
||||
]);
|
||||
expect(handlesOf((await at('live')).body)).toEqual([inApp.handle]);
|
||||
expect(handlesOf((await at('revoked')).body)).toEqual([
|
||||
gone.handle,
|
||||
]);
|
||||
const bad = await at('stopped');
|
||||
expect(bad.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with several recipients', () => {
|
||||
let third: { username: string; uuid: string; id: number };
|
||||
// Holds seeded rows, so no recipient's per-holder cap is what trips.
|
||||
let filler: number;
|
||||
let ownerId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
const username = `kv-batch-${uuidv4().slice(0, 8)}`;
|
||||
await createTestUser(env.server, {
|
||||
username,
|
||||
password: 'pw-test-1234',
|
||||
});
|
||||
const row = await env.server.stores.user.getByUsername(username);
|
||||
third = { username, uuid: row!.uuid as string, id: row!.id! };
|
||||
ownerId = (await env.server.stores.user.getByUsername(
|
||||
env.users.user.username,
|
||||
))!.id!;
|
||||
const fillerName = `kv-filler-${uuidv4().slice(0, 8)}`;
|
||||
await createTestUser(env.server, {
|
||||
username: fillerName,
|
||||
password: 'pw-test-1234',
|
||||
});
|
||||
filler = (await env.server.stores.user.getByUsername(fillerName))!
|
||||
.id!;
|
||||
});
|
||||
|
||||
const shareMany = (body: Record<string, unknown>) =>
|
||||
req('POST', '/kv/shares', env.users.user.token, {
|
||||
prefix: `batch-${uuidv4().slice(0, 8)}:`,
|
||||
...body,
|
||||
});
|
||||
|
||||
it('reports each recipient on its own item, in order', async () => {
|
||||
const res = await shareMany({
|
||||
recipients: [
|
||||
env.users.other.username,
|
||||
'nobody-by-this-name',
|
||||
env.users.user.username,
|
||||
'[email protected]',
|
||||
{ uid: third.uuid },
|
||||
],
|
||||
access: 'write',
|
||||
label: 'Team list',
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const { items } = (await res.json()) as {
|
||||
items: Record<string, unknown>[];
|
||||
};
|
||||
expect(items).toHaveLength(5);
|
||||
expect(items[0]).toMatchObject({
|
||||
recipient: env.users.other.username,
|
||||
handle: expect.any(String),
|
||||
access: 'write',
|
||||
holder: env.users.other.username,
|
||||
label: 'Team list',
|
||||
appUid: null,
|
||||
});
|
||||
expect(items[1]).toEqual({
|
||||
recipient: 'nobody-by-this-name',
|
||||
error: {
|
||||
message: expect.any(String),
|
||||
code: 'subject_does_not_exist',
|
||||
},
|
||||
});
|
||||
expect(items[2]).toMatchObject({
|
||||
error: { code: 'bad_request' },
|
||||
});
|
||||
expect(items[3]).toMatchObject({
|
||||
error: { code: 'invalid_recipient' },
|
||||
});
|
||||
expect(items[4]).toMatchObject({
|
||||
recipient: { uid: third.uuid },
|
||||
holder: third.username,
|
||||
});
|
||||
expect(items[4].handle).not.toBe(items[0].handle);
|
||||
});
|
||||
|
||||
it('answers a recipient named twice with the one share', async () => {
|
||||
const res = await shareMany({
|
||||
recipients: [third.username, { uid: third.uuid }],
|
||||
});
|
||||
const { items } = (await res.json()) as {
|
||||
items: Record<string, unknown>[];
|
||||
};
|
||||
expect(items[0].handle).toEqual(expect.any(String));
|
||||
expect(items[1].handle).toBe(items[0].handle);
|
||||
});
|
||||
|
||||
it('refuses a malformed batch outright', async () => {
|
||||
for (const body of [
|
||||
{ recipients: [] },
|
||||
{ recipients: 'alice' },
|
||||
{
|
||||
recipients: Array.from(
|
||||
{ length: KV_SHARE_BATCH_MAX + 1 },
|
||||
(_, i) => `user-${i}`,
|
||||
),
|
||||
},
|
||||
{
|
||||
recipient: env.users.other.username,
|
||||
recipients: [env.users.other.username],
|
||||
},
|
||||
]) {
|
||||
const res = await shareMany(body);
|
||||
expect(res.status).toBe(400);
|
||||
await expect(res.json()).resolves.toMatchObject({
|
||||
code: 'bad_request',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses the whole batch when it would pass the account cap', async () => {
|
||||
const prefix = `batch-cap-${uuidv4().slice(0, 8)}:`;
|
||||
const existing = await shareMany({
|
||||
prefix,
|
||||
recipients: [env.users.other.username],
|
||||
});
|
||||
const [standing] = (
|
||||
(await existing.json()) as { items: { handle: string }[] }
|
||||
).items;
|
||||
try {
|
||||
const { total } =
|
||||
await env.server.stores.kvShareHandle.countLiveForOwner(
|
||||
ownerId,
|
||||
);
|
||||
// One slot left.
|
||||
await seedKvShareHandles(env.server, {
|
||||
ownerUserId: ownerId,
|
||||
ownerUuid,
|
||||
granteeUserId: filler,
|
||||
count: EVENTS_KV_HANDLES_PER_USER.limit - total - 1,
|
||||
appUid: () => KV_GLOBAL_APP_KEY,
|
||||
});
|
||||
|
||||
const refused = await shareMany({
|
||||
prefix: `batch-over-${uuidv4().slice(0, 8)}:`,
|
||||
recipients: [env.users.other.username, third.username],
|
||||
});
|
||||
expect(refused.status).toBe(409);
|
||||
await expect(refused.json()).resolves.toMatchObject({
|
||||
code: 'events_kv_handle_limit_reached',
|
||||
});
|
||||
expect(
|
||||
(
|
||||
await env.server.stores.kvShareHandle.countLiveForOwner(
|
||||
ownerId,
|
||||
)
|
||||
).total,
|
||||
).toBe(EVENTS_KV_HANDLES_PER_USER.limit - 1);
|
||||
|
||||
// The standing share takes no slot, so one new one still fits.
|
||||
const fits = await shareMany({
|
||||
prefix,
|
||||
recipients: [env.users.other.username, third.username],
|
||||
});
|
||||
expect(fits.status).toBe(200);
|
||||
const { items } = (await fits.json()) as {
|
||||
items: Record<string, unknown>[];
|
||||
};
|
||||
expect(items[0].handle).toBe(standing.handle);
|
||||
expect(items[1]).toMatchObject({ holder: third.username });
|
||||
} finally {
|
||||
await env.server.clients.db.write(
|
||||
'DELETE FROM `kv_share_handles` WHERE `owner_user_id` = ?',
|
||||
[ownerId],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('spends the grant budget per recipient, all or nothing', async () => {
|
||||
const [minute] = KV_SHARE_GRANT_LIMITS;
|
||||
await claimRateLimit(
|
||||
`${minute.scope}:${ownerId}`,
|
||||
minute.limit,
|
||||
minute.window,
|
||||
minute.limit - 1,
|
||||
);
|
||||
try {
|
||||
const refused = await shareMany({
|
||||
recipients: [env.users.other.username, third.username],
|
||||
});
|
||||
expect(refused.status).toBe(429);
|
||||
await expect(refused.json()).resolves.toMatchObject({
|
||||
code: 'too_many_requests',
|
||||
});
|
||||
} finally {
|
||||
await resetBudgets(env);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('/events/kv-handles beside /kv/shares', () => {
|
||||
|
||||
@@ -22,27 +22,99 @@ import type { Actor } from '../../core/actor.js';
|
||||
import { Controller, Delete, Get, Post } from '../../core/http/decorators.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { bodyRecord } from '../../core/http/requestBody.js';
|
||||
import { parseKvShareRecipient } from '../../services/kvShare/kvShares.js';
|
||||
import type { KvShareHandleListOptions } from '../../stores/events/KvShareHandleStore.js';
|
||||
import { KV_HANDLE_LIST_LIMIT_CAP } from '../../stores/events/KvShareHandleStore.js';
|
||||
import {
|
||||
assertShareableAppUid,
|
||||
parseKvShareRecipient,
|
||||
} from '../../services/kvShare/kvShares.js';
|
||||
import type {
|
||||
KvShareHandleListOptions,
|
||||
KvShareOwnerListOptions,
|
||||
KvShareOwnerListState,
|
||||
} from '../../stores/events/KvShareHandleStore.js';
|
||||
import {
|
||||
KV_HANDLE_LIST_LIMIT_CAP,
|
||||
KV_SHARE_OWNER_LIST_STATES,
|
||||
} from '../../stores/events/KvShareHandleStore.js';
|
||||
import { KV_GLOBAL_APP_KEY } from '../../stores/systemKv/SystemKVStore.js';
|
||||
import { normalizeLimit } from '../../util/pagination.js';
|
||||
import { EVENTS_LIST_LIMIT } from '../events/limits.js';
|
||||
import type { KvShareView } from '../../services/kvShare/KvShareService.js';
|
||||
import { EVENTS_LIST_LIMIT, KV_SHARE_BATCH_MAX } from '../events/limits.js';
|
||||
import { PuterController } from '../types.js';
|
||||
|
||||
/** The app-less namespace is no app, so the wire says `null` for it. */
|
||||
const publicAppUid = (appUid: string): string | null =>
|
||||
appUid === KV_GLOBAL_APP_KEY ? null : appUid;
|
||||
|
||||
/**
|
||||
* The namespace a listing's `appUid` names, in the form responses use: an app
|
||||
* uid, or `null` for the app-less namespace.
|
||||
*/
|
||||
const listAppUid = (value: unknown): string | undefined => {
|
||||
if (value === undefined || value === '') return undefined;
|
||||
if (typeof value !== 'string')
|
||||
throw new HttpError(400, '`appUid` must be a string', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
return value === 'null' ? KV_GLOBAL_APP_KEY : assertShareableAppUid(value);
|
||||
};
|
||||
|
||||
const badRequest = (message: string): HttpError =>
|
||||
new HttpError(400, message, { legacyCode: 'bad_request' });
|
||||
|
||||
/** A share as `POST /kv/shares` reports it. */
|
||||
const shareBody = (share: KvShareView) => ({
|
||||
handle: share.handle,
|
||||
prefix: share.prefix,
|
||||
access: share.access,
|
||||
appUid: publicAppUid(share.appUid),
|
||||
holder: share.holderUsername,
|
||||
label: share.label,
|
||||
createdAt: share.createdAt,
|
||||
});
|
||||
|
||||
/** `recipients` when the body names it: 1 to `KV_SHARE_BATCH_MAX` entries. */
|
||||
const batchRecipients = (
|
||||
body: Record<string, unknown>,
|
||||
): unknown[] | undefined => {
|
||||
const { recipient, recipients } = body;
|
||||
if (recipients === undefined) return undefined;
|
||||
if (recipient !== undefined)
|
||||
throw badRequest('Send either `recipient` or `recipients`, not both');
|
||||
if (
|
||||
!Array.isArray(recipients) ||
|
||||
recipients.length === 0 ||
|
||||
recipients.length > KV_SHARE_BATCH_MAX
|
||||
)
|
||||
throw badRequest(
|
||||
`\`recipients\` must be an array of 1 to ${KV_SHARE_BATCH_MAX} recipients`,
|
||||
);
|
||||
return recipients;
|
||||
};
|
||||
|
||||
const listState = (value: unknown): KvShareOwnerListState | undefined => {
|
||||
if (value === undefined || value === '') return undefined;
|
||||
if (!(KV_SHARE_OWNER_LIST_STATES as readonly unknown[]).includes(value))
|
||||
throw new HttpError(400, '`state` must be `live`, `revoked` or `all`', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
return value as KvShareOwnerListState;
|
||||
};
|
||||
|
||||
const pageOptions = (req: Request): KvShareHandleListOptions => {
|
||||
const query = (req.query ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
limit: normalizeLimit(query.limit, { cap: KV_HANDLE_LIST_LIMIT_CAP }),
|
||||
cursor: typeof query.cursor === 'string' ? query.cursor : undefined,
|
||||
includeTotal: query.includeTotal === 'true',
|
||||
appUid: listAppUid(query.appUid),
|
||||
};
|
||||
};
|
||||
|
||||
const ownerPageOptions = (req: Request): KvShareOwnerListOptions => ({
|
||||
...pageOptions(req),
|
||||
state: listState((req.query ?? {}).state),
|
||||
});
|
||||
|
||||
/**
|
||||
* Sharing a region of one account's key-value data with another account.
|
||||
*
|
||||
@@ -56,7 +128,8 @@ export class KvShareController extends PuterController {
|
||||
/**
|
||||
* POST /kv/shares — share a region with another account, `read` unless the
|
||||
* body asks otherwise. Sharing the same region with the same account again
|
||||
* changes its level or label in place.
|
||||
* changes its level or label in place. `recipients` shares it with several
|
||||
* at once and reports each one on its own item.
|
||||
*/
|
||||
@Post('/shares', {
|
||||
subdomain: 'api',
|
||||
@@ -67,29 +140,52 @@ export class KvShareController extends PuterController {
|
||||
async share(req: Request, res: Response): Promise<void> {
|
||||
const actor = this.#requireActor(req);
|
||||
const body = bodyRecord(req);
|
||||
const fields = {
|
||||
prefix: body.prefix,
|
||||
access: body.access,
|
||||
appUid: body.appUid,
|
||||
label: body.label,
|
||||
};
|
||||
const recipients = batchRecipients(body);
|
||||
if (recipients) {
|
||||
const items = await this.services.kvShare.mintKvHandles(
|
||||
actor,
|
||||
{ ...fields, recipients },
|
||||
{ defaultAccess: 'read' },
|
||||
);
|
||||
res.json({
|
||||
items: items.map((item) =>
|
||||
item.share
|
||||
? {
|
||||
recipient: item.recipient,
|
||||
...shareBody(item.share),
|
||||
}
|
||||
: {
|
||||
recipient: item.recipient,
|
||||
error: {
|
||||
message: item.error?.message ?? 'Not shared',
|
||||
code:
|
||||
item.error?.legacyCode ??
|
||||
item.error?.code ??
|
||||
'internal_error',
|
||||
},
|
||||
},
|
||||
),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const share = await this.services.kvShare.mintKvHandle(
|
||||
actor,
|
||||
{
|
||||
prefix: body.prefix,
|
||||
recipient: parseKvShareRecipient(body.recipient),
|
||||
access: body.access,
|
||||
appUid: body.appUid,
|
||||
label: body.label,
|
||||
},
|
||||
{ ...fields, recipient: parseKvShareRecipient(body.recipient) },
|
||||
{ defaultAccess: 'read' },
|
||||
);
|
||||
res.json({
|
||||
handle: share.handle,
|
||||
prefix: share.prefix,
|
||||
access: share.access,
|
||||
appUid: publicAppUid(share.appUid),
|
||||
holder: share.holderUsername,
|
||||
label: share.label,
|
||||
createdAt: share.createdAt,
|
||||
});
|
||||
res.json(shareBody(share));
|
||||
}
|
||||
|
||||
/** GET /kv/shares — what this account has shared out, retired included. */
|
||||
/**
|
||||
* GET /kv/shares — what this account has shared out, retired included
|
||||
* unless `state` narrows it.
|
||||
*/
|
||||
@Get('/shares', {
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
@@ -100,7 +196,7 @@ export class KvShareController extends PuterController {
|
||||
const actor = this.#requireActor(req);
|
||||
const page = await this.services.kvShare.listKvHandles(
|
||||
actor,
|
||||
pageOptions(req),
|
||||
ownerPageOptions(req),
|
||||
);
|
||||
res.json({
|
||||
items: page.items.map((row) => ({
|
||||
|
||||
@@ -151,6 +151,12 @@ describe('StaticAssetsController client_libs_root', () => {
|
||||
|
||||
const sentPutility = await callGetHandler(router, '/putility/v1');
|
||||
expect(sentPutility).toEqual({ filename: 'putility.js/v1.js', root });
|
||||
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
expect(findRoute(router, 'get', route)?.options.subdomain).toBe('js');
|
||||
const sentWorker = await callGetHandler(router, route);
|
||||
expect(sentWorker).toEqual({ filename: 'puter.js/push-sw.js', root });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not register the client_libs routes when the root is unset', () => {
|
||||
@@ -186,6 +192,11 @@ describe('StaticAssetsController puterjs_root', () => {
|
||||
expect(v1.root).toBe(root);
|
||||
const bareV1 = await callGetHandler(router, '/v1');
|
||||
expect(bareV1.filename).toBe('puter.js');
|
||||
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
const worker = await callGetHandler(router, route);
|
||||
expect(worker).toEqual({ filename: 'push-sw.js', root });
|
||||
}
|
||||
});
|
||||
|
||||
it('serves puter.dev.js when present (dev webpack build)', async () => {
|
||||
|
||||
@@ -26,9 +26,9 @@ import type { PuterRouter } from '../../core/http/PuterRouter';
|
||||
/**
|
||||
* Static asset routes.
|
||||
*
|
||||
* /puter.js/v1, /puter.js/v2 → any subdomain /v1, /v2, /putility/v1 → js
|
||||
* subdomain /sdk/* → root subdomain — puter-js bundle /dist/_, /src/_,
|
||||
* /assets/* → root subdomain
|
||||
* /puter.js/v1, /puter.js/v2 → any subdomain /v1, /v2, /push-sw.js,
|
||||
* /putility/v1 → js subdomain /sdk/* → root subdomain — puter-js bundle
|
||||
* /dist/_, /src/_, /assets/* → root subdomain
|
||||
*
|
||||
* Each block depends on its config root (`client_libs_root`, `gui_assets_root`,
|
||||
* `puterjs_root`). When unset, that block is skipped — deployments that don't
|
||||
@@ -55,6 +55,12 @@ export class StaticAssetsController extends PuterController {
|
||||
router.get('/putility/v1', { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile('putility.js/v1.js', { root });
|
||||
});
|
||||
// `/v2/push-sw.js` is the path older docs gave.
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
router.get(route, { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile('puter.js/push-sw.js', { root });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// puter-js SDK mount. GUI loads it at `/sdk/puter.dev.js`; the
|
||||
@@ -100,6 +106,11 @@ export class StaticAssetsController extends PuterController {
|
||||
router.get('/v2', { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile(puterJsFile, { root: puterjsRoot });
|
||||
});
|
||||
for (const route of ['/push-sw.js', '/v2/push-sw.js']) {
|
||||
router.get(route, { subdomain: 'js' }, (_req, res) => {
|
||||
res.sendFile('push-sw.js', { root: puterjsRoot });
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import {
|
||||
setupPuterTestEnv,
|
||||
TEST_OTHER_USER_CREDENTIALS,
|
||||
TEST_USER_CREDENTIALS,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
|
||||
/** PROPFIND has no paging, so a folder must list whole however large. */
|
||||
describe('PROPFIND on a folder larger than one query', () => {
|
||||
let env: PuterTestEnv;
|
||||
const total = 10_050;
|
||||
let dir: string;
|
||||
let dirUuid: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
const { username } = env.users.user;
|
||||
dir = `/${username}/Documents/big-${randomUUID().slice(0, 8)}`;
|
||||
const owner = (await env.server.stores.user.getByUsername(username))!;
|
||||
const parent = (await env.server.stores.fsEntry.getEntryByPath(
|
||||
`/${username}/Documents`,
|
||||
))!;
|
||||
dirUuid = randomUUID();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const insert = (rows: unknown[][]) =>
|
||||
env.server.clients.db.write(
|
||||
`INSERT INTO fsentries (uuid, user_id, parent_id, parent_uid, name, path, is_dir, modified, created, accessed, size) VALUES ${rows.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
|
||||
rows.flat(),
|
||||
);
|
||||
await insert([
|
||||
[
|
||||
dirUuid,
|
||||
owner.id,
|
||||
parent.id,
|
||||
parent.uuid,
|
||||
dir.split('/').at(-1),
|
||||
dir,
|
||||
1,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
0,
|
||||
],
|
||||
]);
|
||||
const folder = (await env.server.stores.fsEntry.getEntryByPath(dir))!;
|
||||
for (let start = 0; start < total; start += 500) {
|
||||
const rows: unknown[][] = [];
|
||||
for (let i = start; i < Math.min(total, start + 500); i++) {
|
||||
const name = `f${String(i).padStart(5, '0')}.txt`;
|
||||
rows.push([
|
||||
randomUUID(),
|
||||
owner.id,
|
||||
folder.id,
|
||||
folder.uuid,
|
||||
name,
|
||||
`${dir}/${name}`,
|
||||
0,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
0,
|
||||
]);
|
||||
}
|
||||
await insert(rows);
|
||||
}
|
||||
|
||||
const actor = { user: owner };
|
||||
await runWithContext({ actor: actor as never }, () =>
|
||||
env.server.services.share.share(actor as never, {
|
||||
uid: dirUuid,
|
||||
recipient: { username: env.users.other.username },
|
||||
mode: 'read',
|
||||
}),
|
||||
);
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const propfind = async (
|
||||
path: string,
|
||||
credentials: { username: string; password: string },
|
||||
) => {
|
||||
const port = new URL(env.apiOrigin).port;
|
||||
const auth = Buffer.from(
|
||||
`${credentials.username}:${credentials.password}`,
|
||||
).toString('base64');
|
||||
const response = await fetch(
|
||||
`http://dav.puter.localhost:${port}${encodeURI(path)}`,
|
||||
{
|
||||
method: 'PROPFIND',
|
||||
headers: { Authorization: `Basic ${auth}`, Depth: '1' },
|
||||
},
|
||||
);
|
||||
return { status: response.status, body: await response.text() };
|
||||
};
|
||||
|
||||
const hrefsOf = (xml: string) =>
|
||||
[...xml.matchAll(/<D:href>([^<]*)<\/D:href>/g)].map((m) => m[1]!);
|
||||
|
||||
it('lists every child', async () => {
|
||||
const { status, body } = await propfind(dir, TEST_USER_CREDENTIALS);
|
||||
expect(status).toBe(207);
|
||||
expect(body.endsWith('</D:multistatus>')).toBe(true);
|
||||
const hrefs = hrefsOf(body);
|
||||
expect(hrefs).toHaveLength(1 + total);
|
||||
expect(new Set(hrefs).size).toBe(1 + total);
|
||||
expect(body).toContain(`${dir}/f10049.txt`);
|
||||
}, 60_000);
|
||||
|
||||
it("masks every page of a share recipient's listing", async () => {
|
||||
const masked = `/${env.users.user.username}/${dirUuid}/${dir.split('/').at(-1)}`;
|
||||
const { status, body } = await propfind(
|
||||
masked,
|
||||
TEST_OTHER_USER_CREDENTIALS,
|
||||
);
|
||||
expect(status).toBe(207);
|
||||
const hrefs = hrefsOf(body);
|
||||
expect(hrefs).toHaveLength(1 + total);
|
||||
expect(hrefs.every((href) => href.startsWith(masked))).toBe(true);
|
||||
expect(body).not.toContain('/Documents/');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -24,6 +24,7 @@ import { posix as pathPosix } from 'node:path';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { pipeToResponse } from '../../core/http/pipeToResponse.js';
|
||||
import {
|
||||
assertNotSuspended,
|
||||
assertTeam2fa,
|
||||
@@ -58,6 +59,7 @@ import {
|
||||
import { Context } from '../../core/context.js';
|
||||
import type { FSEntry } from '../../stores/fs/FSEntry.js';
|
||||
import {
|
||||
contextBoundStream,
|
||||
emitGuiItemEvent,
|
||||
type GuiItemEventName,
|
||||
} from '../fs/legacyFsHelpers.js';
|
||||
@@ -600,19 +602,41 @@ export class WebDAVController extends PuterController {
|
||||
propfindEntry(maskEntryPath(entry), entry, entry.isDir),
|
||||
];
|
||||
|
||||
if (depth !== '0' && entry.isDir) {
|
||||
const children = await this.services.fs.listDirectory(
|
||||
entry.uuid,
|
||||
{},
|
||||
);
|
||||
for (const child of children) {
|
||||
responses.push(
|
||||
propfindEntry(maskEntryPath(child), child, child.isDir),
|
||||
);
|
||||
}
|
||||
if (depth === '0' || !entry.isDir) {
|
||||
this.#sendMultistatus(res, responses);
|
||||
return;
|
||||
}
|
||||
|
||||
this.#sendMultistatus(res, responses);
|
||||
const toXml = (children: FSEntry[]) =>
|
||||
children
|
||||
.map((child) =>
|
||||
propfindEntry(maskEntryPath(child), child, child.isDir),
|
||||
)
|
||||
.join('\n');
|
||||
const pages = this.services.fs.listDirectoryPages(entry.uuid);
|
||||
const first = await pages.next();
|
||||
if (!first.done && first.value.length > 0)
|
||||
responses.push(toXml(first.value));
|
||||
const second = await pages.next();
|
||||
if (second.done) {
|
||||
this.#sendMultistatus(res, responses);
|
||||
return;
|
||||
}
|
||||
|
||||
// Past one page the listing streams, so a large folder isn't held in
|
||||
// memory whole. A failure partway cuts the response off.
|
||||
const secondPage = second.value;
|
||||
async function* body() {
|
||||
yield `${MULTISTATUS_OPEN}${responses.join('\n')}`;
|
||||
yield `\n${toXml(secondPage)}`;
|
||||
for await (const page of pages)
|
||||
if (page.length > 0) yield `\n${toXml(page)}`;
|
||||
yield MULTISTATUS_CLOSE;
|
||||
}
|
||||
res.status(207).set({
|
||||
'Content-Type': 'application/xml; charset=utf-8',
|
||||
});
|
||||
await pipeToResponse(contextBoundStream(body()), res);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1289,8 +1313,12 @@ function escapeXml(text: string): string {
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
const MULTISTATUS_OPEN =
|
||||
'<?xml version="1.0" encoding="utf-8"?>\n<D:multistatus xmlns:D="DAV:">\n';
|
||||
const MULTISTATUS_CLOSE = '\n</D:multistatus>';
|
||||
|
||||
function wrapMultistatus(inner: string): string {
|
||||
return `<?xml version="1.0" encoding="utf-8"?>\n<D:multistatus xmlns:D="DAV:">\n${inner}\n</D:multistatus>`;
|
||||
return `${MULTISTATUS_OPEN}${inner}${MULTISTATUS_CLOSE}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -63,10 +63,17 @@ export interface KnownContextFields {
|
||||
driverName: string;
|
||||
/**
|
||||
* Aborts when the client disconnects before the response has finished (set
|
||||
* by DriverController). Long-running drivers poll it so work nobody will
|
||||
* receive stops early and is never metered.
|
||||
* by DriverController). Some drivers (Gemini, Replicate) stop early on it.
|
||||
* Async video and image jobs keep polling and are metered after a
|
||||
* disconnect, since the provider bills them either way.
|
||||
*/
|
||||
abortSignal: AbortSignal;
|
||||
/**
|
||||
* Keeps the call's concurrency slot until `work` settles (set by
|
||||
* DriverController when the method is capped), for work that outlives the
|
||||
* response.
|
||||
*/
|
||||
keepDriverSlot: (work: Promise<unknown>) => void;
|
||||
/**
|
||||
* Aborts the chat driver's upstream request. A client disconnect doesn't
|
||||
* fire it: the upstream bills a request it has accepted either way, so the
|
||||
@@ -90,6 +97,7 @@ const KNOWN_KEY_FLAGS: Record<keyof KnownContextFields, true> = {
|
||||
requestId: true,
|
||||
driverName: true,
|
||||
abortSignal: true,
|
||||
keepDriverSlot: true,
|
||||
upstreamSignal: true,
|
||||
strictUpstreamErrors: true,
|
||||
};
|
||||
|
||||
@@ -476,9 +476,8 @@ function actorMatchesExpectedApp(
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror of `resolvePrivateIdentity` for public hosted apps. Reads the sticky
|
||||
* `puter.public.hosted.actor.token` cookie first, then the same
|
||||
* session/bootstrap fallbacks.
|
||||
* Fresh authenticated identities replace the sticky public billing cookie when
|
||||
* the visitor switches accounts.
|
||||
*/
|
||||
export async function resolvePublicHostedIdentity(opts: {
|
||||
req: Request;
|
||||
@@ -500,37 +499,6 @@ export async function resolvePublicHostedIdentity(opts: {
|
||||
const cookies = (req as Request & { cookies?: Record<string, string> })
|
||||
.cookies;
|
||||
|
||||
const { name: publicCookieNameV2, legacyName: publicCookieNameLegacy } =
|
||||
HOSTED_ASSET_COOKIES.public;
|
||||
const publicCookieToken =
|
||||
(typeof cookies?.[publicCookieNameV2] === 'string'
|
||||
? cookies[publicCookieNameV2]
|
||||
: null) ??
|
||||
(typeof cookies?.[publicCookieNameLegacy] === 'string'
|
||||
? cookies[publicCookieNameLegacy]
|
||||
: null);
|
||||
if (publicCookieToken) {
|
||||
try {
|
||||
const claims = await authService.verifyHostedAssetToken(
|
||||
'public',
|
||||
publicCookieToken,
|
||||
{
|
||||
appUid: expectedAppUid,
|
||||
subdomain: expectedSubdomain,
|
||||
host: expectedHost,
|
||||
},
|
||||
);
|
||||
return {
|
||||
source: 'public-cookie',
|
||||
userUid: claims.userUid,
|
||||
sessionUuid: claims.sessionUuid,
|
||||
hasValidPublicCookie: true,
|
||||
};
|
||||
} catch {
|
||||
/* fall through — v1-signed cookies land here and get re-minted */
|
||||
}
|
||||
}
|
||||
|
||||
const existingActor = req.actor;
|
||||
if (
|
||||
existingActor?.user?.uuid &&
|
||||
@@ -586,6 +554,37 @@ export async function resolvePublicHostedIdentity(opts: {
|
||||
}
|
||||
}
|
||||
|
||||
const { name: publicCookieNameV2, legacyName: publicCookieNameLegacy } =
|
||||
HOSTED_ASSET_COOKIES.public;
|
||||
const publicCookieToken =
|
||||
(typeof cookies?.[publicCookieNameV2] === 'string'
|
||||
? cookies[publicCookieNameV2]
|
||||
: null) ??
|
||||
(typeof cookies?.[publicCookieNameLegacy] === 'string'
|
||||
? cookies[publicCookieNameLegacy]
|
||||
: null);
|
||||
if (publicCookieToken) {
|
||||
try {
|
||||
const claims = await authService.verifyHostedAssetToken(
|
||||
'public',
|
||||
publicCookieToken,
|
||||
{
|
||||
appUid: expectedAppUid,
|
||||
subdomain: expectedSubdomain,
|
||||
host: expectedHost,
|
||||
},
|
||||
);
|
||||
return {
|
||||
source: 'public-cookie',
|
||||
userUid: claims.userUid,
|
||||
sessionUuid: claims.sessionUuid,
|
||||
hasValidPublicCookie: true,
|
||||
};
|
||||
} catch {
|
||||
/* fall through — v1-signed cookies land here and get re-minted */
|
||||
}
|
||||
}
|
||||
|
||||
return { source: 'none' };
|
||||
}
|
||||
|
||||
|
||||
@@ -2591,3 +2591,239 @@ describe('createPuterSiteMiddleware — private-app gate decisions', () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('public hosted asset billing identity', () => {
|
||||
it.each(['actor', 'session', 'query'] as const)(
|
||||
'replaces an older billing cookie with the current %s identity',
|
||||
async (source) => {
|
||||
const owner = await makeUserWithHome();
|
||||
const previous = await makeUser();
|
||||
const current = await makeUser();
|
||||
const home = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}`,
|
||||
))!;
|
||||
const sub = `switch-${uuidv4().slice(0, 8)}`;
|
||||
const host = `${sub}.site.puter.localhost`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: home.id,
|
||||
});
|
||||
await writeFile(
|
||||
owner.id,
|
||||
`/${owner.username}/index.html`,
|
||||
Buffer.from('hello'),
|
||||
);
|
||||
const app = await server.stores.app.create(
|
||||
{ name: sub, title: sub, index_url: `http://${host}/` },
|
||||
{ ownerUserId: owner.id },
|
||||
);
|
||||
const previousToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(previous),
|
||||
app.uid,
|
||||
);
|
||||
const previousActor =
|
||||
(await server.services.auth.authenticateFromToken(
|
||||
previousToken,
|
||||
))!;
|
||||
const currentToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(current),
|
||||
app.uid,
|
||||
);
|
||||
const oldCookie = await server.services.auth.createHostedAssetToken(
|
||||
'public',
|
||||
{
|
||||
appUid: app.uid,
|
||||
userUid: previous.uuid,
|
||||
sessionUuid: previousActor.session?.uid,
|
||||
subdomain: sub,
|
||||
host,
|
||||
},
|
||||
);
|
||||
const cookieName = HOSTED_ASSET_COOKIES.public.name;
|
||||
const req = makeReq({
|
||||
hostname: host,
|
||||
path: '/index.html',
|
||||
cookies: {
|
||||
[cookieName]: oldCookie,
|
||||
...(source === 'session' ? { session: currentToken } : {}),
|
||||
},
|
||||
query:
|
||||
source === 'query'
|
||||
? { 'puter.auth.token': currentToken }
|
||||
: {},
|
||||
});
|
||||
if (source === 'actor')
|
||||
req.actor =
|
||||
(await server.services.auth.authenticateFromToken(
|
||||
currentToken,
|
||||
))!;
|
||||
const { res, out } = makeRes();
|
||||
await buildMiddleware({ cookie_name: 'session' })(
|
||||
req,
|
||||
res,
|
||||
vi.fn(),
|
||||
);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(out.statusCode).toBe(200);
|
||||
expect(req.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
const refreshed = out.cookies[cookieName];
|
||||
expect(refreshed).toBeTruthy();
|
||||
expect(
|
||||
(
|
||||
await server.services.auth.verifyHostedAssetToken(
|
||||
'public',
|
||||
refreshed,
|
||||
{ appUid: app.uid, host },
|
||||
)
|
||||
).userUid,
|
||||
).toBe(current.uuid);
|
||||
const nextReq = makeReq({
|
||||
hostname: host,
|
||||
path: '/index.html',
|
||||
cookies: { [cookieName]: refreshed },
|
||||
});
|
||||
await runMiddleware(buildMiddleware(), nextReq);
|
||||
expect(nextReq.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
},
|
||||
);
|
||||
|
||||
it('bills recognized visitors, reuses the parent session, and falls back to the owner', async () => {
|
||||
const owner = await makeUserWithHome();
|
||||
const visitor = await makeUser();
|
||||
const home = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}`,
|
||||
))!;
|
||||
const sub = `egress-${uuidv4().slice(0, 8)}`;
|
||||
const host = `${sub}.site.puter.localhost`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: home.id,
|
||||
});
|
||||
await writeFile(
|
||||
owner.id,
|
||||
`/${owner.username}/index.html`,
|
||||
Buffer.from('hello'),
|
||||
);
|
||||
const app = await server.stores.app.create(
|
||||
{ name: sub, title: sub, index_url: `http://${host}/` },
|
||||
{ ownerUserId: owner.id },
|
||||
);
|
||||
const token = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
app.uid,
|
||||
);
|
||||
const authenticated =
|
||||
await server.services.auth.authenticateFromToken(token);
|
||||
const serve = async (query = {}, cookies = {}, headers = {}) => {
|
||||
const req = makeReq({ hostname: host, path: '/index.html', query });
|
||||
Object.assign(req, { cookies, headers });
|
||||
const { res, out } = makeRes();
|
||||
await buildMiddleware()(req, res, vi.fn());
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(out.statusCode).toBe(200);
|
||||
return { req, out };
|
||||
};
|
||||
const create = vi.spyOn(server.stores.session, 'create');
|
||||
try {
|
||||
const first = await serve({ 'puter.auth.token': token });
|
||||
expect(first.req.egressActor?.user?.uuid).toBe(visitor.uuid);
|
||||
const cookieName = HOSTED_ASSET_COOKIES.public.name;
|
||||
const cookie = first.out.cookies[cookieName];
|
||||
expect(cookie).toBeTruthy();
|
||||
const claims = await server.services.auth.verifyHostedAssetToken(
|
||||
'public',
|
||||
cookie,
|
||||
{ appUid: app.uid, host },
|
||||
);
|
||||
expect(claims.sessionUuid).toBe(authenticated?.session?.uid);
|
||||
const next = await serve({}, { [cookieName]: cookie });
|
||||
expect(next.req.egressActor?.user?.uuid).toBe(visitor.uuid);
|
||||
expect(next.out.cookies).toEqual({});
|
||||
const blockedCookies = await serve(
|
||||
{},
|
||||
{},
|
||||
{ referer: `http://${host}/?puter.auth.token=${token}` },
|
||||
);
|
||||
expect(blockedCookies.req.egressActor?.user?.uuid).toBe(
|
||||
visitor.uuid,
|
||||
);
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
const anonymous = await serve();
|
||||
expect(anonymous.req.egressActor?.user?.uuid).toBe(owner.uuid);
|
||||
const invalid = await serve({}, { [cookieName]: 'invalid' });
|
||||
expect(invalid.req.egressActor?.user?.uuid).toBe(owner.uuid);
|
||||
await server.services.auth.revokeAllSessionsForUserId(visitor.id);
|
||||
const revoked = await serve({}, { [cookieName]: cookie });
|
||||
expect(revoked.req.egressActor?.user?.uuid).toBe(owner.uuid);
|
||||
} finally {
|
||||
create.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('site rooted in the owner tree, root moved to Trash', () => {
|
||||
it('stops serving a trashed folder and resumes when restored', async () => {
|
||||
const owner = await makeUserWithHome();
|
||||
const dir = await server.services.fs.mkdir(owner.id, {
|
||||
path: `/${owner.username}/Documents/site`,
|
||||
createMissingParents: true,
|
||||
});
|
||||
const body = Buffer.from('<html>trashed</html>');
|
||||
await server.services.fs.write(owner.id, {
|
||||
fileMetadata: {
|
||||
path: `/${owner.username}/Documents/site/index.html`,
|
||||
size: body.byteLength,
|
||||
},
|
||||
fileContent: body,
|
||||
});
|
||||
const sub = `tr-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: dir.id,
|
||||
});
|
||||
const serveIndex = async () => {
|
||||
const mw = createPuterSiteMiddleware(hostingConfig, {
|
||||
clients: server.clients,
|
||||
stores: server.stores,
|
||||
services: server.services,
|
||||
});
|
||||
const { res, out } = makeRes();
|
||||
await mw(
|
||||
makeReq({
|
||||
hostname: `${sub}.site.puter.localhost`,
|
||||
path: '/',
|
||||
}),
|
||||
res,
|
||||
vi.fn(),
|
||||
);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
return out;
|
||||
};
|
||||
expect((await serveIndex()).statusCode).toBe(200);
|
||||
|
||||
const trash = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}/Trash`,
|
||||
))!;
|
||||
const entry = (await server.stores.fsEntry.getEntryById(dir.id))!;
|
||||
await server.services.fs.move(owner.id, {
|
||||
source: entry,
|
||||
destinationParent: trash,
|
||||
});
|
||||
const moved = (await server.stores.fsEntry.getEntryById(dir.id))!;
|
||||
expect(moved.path.startsWith(`/${owner.username}/Trash/`)).toBe(true);
|
||||
|
||||
const out = await serveIndex();
|
||||
expect(out.statusCode).toBe(404);
|
||||
const documents = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}/Documents`,
|
||||
))!;
|
||||
await server.services.fs.move(owner.id, {
|
||||
source: moved,
|
||||
destinationParent: documents,
|
||||
});
|
||||
expect((await serveIndex()).statusCode).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -27,6 +27,7 @@ import { MANAGE_PERM_PREFIX } from '../../../services/permission/consts';
|
||||
import type { puterStores } from '../../../stores';
|
||||
import type { IConfig, LayerInstances } from '../../../types';
|
||||
import { awaitLater } from '../../../util/concurrency';
|
||||
import { isTrashedPath } from '../../../util/hostedAppBacking';
|
||||
import { makeActor } from '../../actor';
|
||||
import {
|
||||
buildAppCenterFallback,
|
||||
@@ -109,10 +110,6 @@ const isServableEntryPath = (
|
||||
return !isWorkersSourcePath(relPath) && !isSiteConfigPath(relPath);
|
||||
};
|
||||
|
||||
/** Inside some owner's top-level Trash, which is where Delete puts things. */
|
||||
const isTrashedPath = (path: string): boolean =>
|
||||
/^\/[^/]+\/Trash(\/|$)/u.test(path);
|
||||
|
||||
/**
|
||||
* Suggested value for `config.hosting_csp`. Not applied unless configured.
|
||||
*
|
||||
@@ -333,6 +330,7 @@ export const createPuterSiteMiddleware = (
|
||||
|
||||
const associatedApp = (await appLookup) as AppRow | null;
|
||||
|
||||
let hostedVisitorUid: string | undefined;
|
||||
const isPrivateApp = Boolean(associatedApp?.is_private);
|
||||
const privateApp = isPrivateApp ? associatedApp : null;
|
||||
|
||||
@@ -400,6 +398,8 @@ export const createPuterSiteMiddleware = (
|
||||
return;
|
||||
}
|
||||
|
||||
hostedVisitorUid = identity.userUid;
|
||||
|
||||
// Entitlement check runs on every request — matching v1. The
|
||||
// sticky cookie is an identity shortcut, not an access cache;
|
||||
// the marketplace extension already caches access decisions
|
||||
@@ -509,11 +509,8 @@ export const createPuterSiteMiddleware = (
|
||||
res.status(404).type('text/plain').send('Subdomain not found');
|
||||
return;
|
||||
} else {
|
||||
// Public hosted site. Mint the public hosted-actor cookie if
|
||||
// we can identify the visitor — lets the hosted page make
|
||||
// cross-origin requests as the actor without needing a
|
||||
// host-scoped main session cookie. No-op for anonymous
|
||||
// visitors.
|
||||
// Remember the authenticated visitor for asset billing across
|
||||
// requests that do not carry the original app token.
|
||||
try {
|
||||
const identity = await resolvePublicHostedIdentity({
|
||||
req,
|
||||
@@ -526,6 +523,7 @@ export const createPuterSiteMiddleware = (
|
||||
expectedSubdomain: subdomain,
|
||||
expectedHost: host,
|
||||
});
|
||||
hostedVisitorUid = identity.userUid;
|
||||
if (
|
||||
identity.userUid &&
|
||||
!(identity as { hasValidPublicCookie?: boolean })
|
||||
@@ -574,7 +572,7 @@ export const createPuterSiteMiddleware = (
|
||||
.send(SUBDOMAIN_404);
|
||||
return;
|
||||
}
|
||||
if (!rootEntry.isDir) {
|
||||
if (!rootEntry.isDir || isTrashedPath(rootEntry.path)) {
|
||||
res.status(404)
|
||||
.type('text/html; charset=UTF-8')
|
||||
.send(SUBDOMAIN_404);
|
||||
@@ -584,37 +582,27 @@ export const createPuterSiteMiddleware = (
|
||||
// A site rooted in someone else's directory stands on the `manage`
|
||||
// grant that authorized publishing it, and serves that whole subtree
|
||||
// with the ACL bypassed — so the grant is re-read on every request.
|
||||
// Trash is its own check: a grant is keyed on the node, not its
|
||||
// location, so it survives the owner deleting the directory. Sites
|
||||
// rooted in their publisher's own tree skip all of this.
|
||||
if (rootEntry.userId !== site.user_id) {
|
||||
let stillPublishable = false;
|
||||
if (!isTrashedPath(rootEntry.path)) {
|
||||
try {
|
||||
stillPublishable = await layers.services.acl.check(
|
||||
makeActor({
|
||||
user: {
|
||||
id: owner.id,
|
||||
uuid: owner.uuid,
|
||||
username: owner.username,
|
||||
},
|
||||
}),
|
||||
{
|
||||
path: rootEntry.path,
|
||||
resolveAncestors: () =>
|
||||
layers.services.fs.getAncestorChain(
|
||||
rootEntry.path,
|
||||
),
|
||||
try {
|
||||
stillPublishable = await layers.services.acl.check(
|
||||
makeActor({
|
||||
user: {
|
||||
id: owner.id,
|
||||
uuid: owner.uuid,
|
||||
username: owner.username,
|
||||
},
|
||||
MANAGE_PERM_PREFIX,
|
||||
);
|
||||
} catch (e) {
|
||||
// Fail closed — an unreadable grant is not a held one.
|
||||
console.warn(
|
||||
'[puter-site] publish grant recheck failed',
|
||||
e,
|
||||
);
|
||||
}
|
||||
}),
|
||||
{
|
||||
path: rootEntry.path,
|
||||
resolveAncestors: () =>
|
||||
layers.services.fs.getAncestorChain(rootEntry.path),
|
||||
},
|
||||
MANAGE_PERM_PREFIX,
|
||||
);
|
||||
} catch (e) {
|
||||
// Fail closed — an unreadable grant is not a held one.
|
||||
console.warn('[puter-site] publish grant recheck failed', e);
|
||||
}
|
||||
if (!stillPublishable) {
|
||||
res.status(404)
|
||||
@@ -837,7 +825,22 @@ export const createPuterSiteMiddleware = (
|
||||
// hosting is unauthenticated by design, so most visitors are nobody in
|
||||
// particular. Set unconditionally because hosting subdomains are not
|
||||
// metered by default: this is what opts the response in.
|
||||
req.egressActor = req.actor ?? {
|
||||
let visitorActor = req.actor;
|
||||
if (hostedVisitorUid && visitorActor?.user?.uuid !== hostedVisitorUid) {
|
||||
try {
|
||||
const visitor =
|
||||
await layers.stores.user.getByUuid(hostedVisitorUid);
|
||||
if (visitor && !visitor.suspended) {
|
||||
visitorActor = makeActor({
|
||||
user: visitor,
|
||||
app: associatedApp ?? undefined,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Unknown requesters are billed to the site owner.
|
||||
}
|
||||
}
|
||||
req.egressActor = visitorActor ?? {
|
||||
user: {
|
||||
uuid: owner.uuid,
|
||||
id: owner.id,
|
||||
|
||||
@@ -1049,8 +1049,8 @@ export function concurrencyGate(opts) {
|
||||
* Acquire a concurrent slot for a driver call. Mirrors `checkDriverRateLimit`
|
||||
* but returns an acquisition handle: caller MUST invoke `release()` on the
|
||||
* returned object exactly once, even on thrown errors — typically in a
|
||||
* `finally`. `ok: false` means the slot was full; callers should reject with
|
||||
* 429 in that case.
|
||||
* `finally`. `renew()` keeps a long-held slot past the orphan TTL. `ok: false`
|
||||
* means the slot was full; callers should reject with 429 in that case.
|
||||
*
|
||||
* `opts` is the resolved per-method spec from the driver's decorator (or
|
||||
* imperative `concurrent` field). Omitting `opts` (driver declares nothing)
|
||||
@@ -1060,7 +1060,7 @@ export function concurrencyGate(opts) {
|
||||
*/
|
||||
export async function acquireDriverConcurrent(req, ifaceName, method, opts) {
|
||||
if (!opts || typeof opts.limit !== 'number') {
|
||||
return { ok: true, release: noop };
|
||||
return { ok: true, release: noop, renew: noop };
|
||||
}
|
||||
const slot = await charge(
|
||||
req,
|
||||
@@ -1072,8 +1072,7 @@ export async function acquireDriverConcurrent(req, ifaceName, method, opts) {
|
||||
label: '[concurrent] driver acquire',
|
||||
},
|
||||
);
|
||||
if (!slot) return { ok: true, release: noop };
|
||||
if (!slot.ok) return { ok: false, release: noop };
|
||||
const { release } = guardSlot(slot, '[concurrent] driver');
|
||||
return { ok: true, release };
|
||||
if (!slot) return { ok: true, release: noop, renew: noop };
|
||||
if (!slot.ok) return { ok: false, release: noop, renew: noop };
|
||||
return guardSlot(slot, '[concurrent] driver');
|
||||
}
|
||||
@@ -104,6 +104,8 @@ const default_user_permissions = {
|
||||
// is there; which app may is the per-app grant, which is what the user is
|
||||
// actually asked about.
|
||||
'events:background': {},
|
||||
// Likewise, the per-app grant decides whose notifications go out as pushes.
|
||||
'notifications:push': {},
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -816,6 +816,9 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
// where "done" is the stream draining rather than this method
|
||||
// returning.
|
||||
let hold: CreditHold = NO_CREDIT_HOLD;
|
||||
// The gate's estimate for the current attempt: the message estimate
|
||||
// plus whatever its provider reads back as prompt beyond the messages.
|
||||
let attemptPromptTokenEstimate = promptTokenEstimate;
|
||||
// The current attempt's balance-bound output cap, if it had one.
|
||||
let fundsCap: number | undefined;
|
||||
// The cost factor the current attempt's charges are recorded at.
|
||||
@@ -824,12 +827,16 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
// charge, so a rate change mid-request can't split the two.
|
||||
let pricingTime = new Date();
|
||||
if (!useFakeProvider) {
|
||||
({ hold, fundsCap, costFactor } = await this.#applyCreditGate(
|
||||
actor,
|
||||
model,
|
||||
args,
|
||||
{ promptTokenEstimate, requestedMaxTokens, pricingTime },
|
||||
));
|
||||
({
|
||||
hold,
|
||||
fundsCap,
|
||||
costFactor,
|
||||
promptTokenEstimate: attemptPromptTokenEstimate,
|
||||
} = await this.#applyCreditGate(actor, model, args, {
|
||||
promptTokenEstimate,
|
||||
requestedMaxTokens,
|
||||
pricingTime,
|
||||
}));
|
||||
}
|
||||
|
||||
// First attempt
|
||||
@@ -957,12 +964,16 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
// this one starts from nothing held.
|
||||
pricingTime = new Date();
|
||||
if (!useFakeProvider) {
|
||||
({ hold, fundsCap, costFactor } =
|
||||
await this.#applyCreditGate(actor, fallback, args, {
|
||||
promptTokenEstimate,
|
||||
requestedMaxTokens,
|
||||
pricingTime,
|
||||
}));
|
||||
({
|
||||
hold,
|
||||
fundsCap,
|
||||
costFactor,
|
||||
promptTokenEstimate: attemptPromptTokenEstimate,
|
||||
} = await this.#applyCreditGate(actor, fallback, args, {
|
||||
promptTokenEstimate,
|
||||
requestedMaxTokens,
|
||||
pricingTime,
|
||||
}));
|
||||
}
|
||||
|
||||
tried.add(routeId(fallback.provider!, fallback.id));
|
||||
@@ -1188,7 +1199,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
actor,
|
||||
chatStream,
|
||||
model,
|
||||
promptTokenEstimate,
|
||||
promptTokenEstimate: attemptPromptTokenEstimate,
|
||||
completionId,
|
||||
username,
|
||||
intendedProvider,
|
||||
@@ -1432,10 +1443,16 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
/** The instant this attempt is priced at. */
|
||||
pricingTime: Date;
|
||||
},
|
||||
): Promise<{ hold: CreditHold; fundsCap?: number; costFactor: number }> {
|
||||
): Promise<{
|
||||
hold: CreditHold;
|
||||
fundsCap?: number;
|
||||
costFactor: number;
|
||||
/** The prompt estimate this attempt was priced on. */
|
||||
promptTokenEstimate: number;
|
||||
}> {
|
||||
const metering = this.services.metering;
|
||||
const { promptTokenEstimate, requestedMaxTokens, pricingTime } =
|
||||
estimates;
|
||||
const { requestedMaxTokens, pricingTime } = estimates;
|
||||
let { promptTokenEstimate } = estimates;
|
||||
// A provider can name its own cost keys for this request (Claude's
|
||||
// fast-mode rate card), cap its output, and size an extra reservation
|
||||
// the per-token math can't see (a web-search/advisor tool that could
|
||||
@@ -1446,6 +1463,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
this.#providers[model.provider!]?.requestPricing?.(args, model, {
|
||||
promptTokenEstimate,
|
||||
}) ?? {};
|
||||
promptTokenEstimate += pricing.additionalPromptTokens ?? 0;
|
||||
const { inputKey: defaultInputKey, outputKey: defaultOutputKey } =
|
||||
costKeys(model);
|
||||
const inputKey = pricing.inputKey ?? defaultInputKey;
|
||||
@@ -1560,7 +1578,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
actor,
|
||||
Math.max(worstCaseCost, minimumCredits),
|
||||
);
|
||||
return { hold, fundsCap, costFactor };
|
||||
return { hold, fundsCap, costFactor, promptTokenEstimate };
|
||||
}
|
||||
|
||||
/** The key `model`'s usage is recorded and cost-factored under. */
|
||||
|
||||
@@ -194,7 +194,9 @@ export class OpenAICompatProvider implements IChatProvider {
|
||||
await OpenAIUtil.process_input_messages(messages, {
|
||||
preserveToolCallOrder: this.preservesToolCallOrder(model),
|
||||
keepExtraContent: this.keepsToolCallExtraContent(model),
|
||||
keepAssistantContent: this.preservesAssistantContent(model),
|
||||
}),
|
||||
model,
|
||||
);
|
||||
|
||||
const tools = args.tools
|
||||
@@ -324,8 +326,19 @@ export class OpenAICompatProvider implements IChatProvider {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a replayed assistant turn keeps its text beside hoisted tool
|
||||
* calls.
|
||||
*/
|
||||
protected preservesAssistantContent(_model: IChatModel): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Last pass over the messages, after `process_input_messages`. */
|
||||
protected prepareMessages(messages: PuterMessage[]): PuterMessage[] {
|
||||
protected prepareMessages(
|
||||
messages: PuterMessage[],
|
||||
_model: IChatModel,
|
||||
): PuterMessage[] {
|
||||
return messages;
|
||||
}
|
||||
|
||||
|
||||
+33
-2
@@ -20,8 +20,8 @@
|
||||
/**
|
||||
* Integration test for the Moonshot provider.
|
||||
*
|
||||
* Uses `kimi-k2.6` (the cheapest current Kimi model). Skipped
|
||||
* when `PUTER_TEST_AI_MOONSHOT_API_KEY` is unset.
|
||||
* Uses `kimi-k2.6` (the cheapest current Kimi model) and `kimi-k3.1-pro`.
|
||||
* Skipped when `PUTER_TEST_AI_MOONSHOT_API_KEY` is unset.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
@@ -59,5 +59,36 @@ describe.skipIf(skipUnlessEnv(ENV_VAR))(
|
||||
?.content;
|
||||
expect(typeof text === 'string' && text.length > 0).toBe(true);
|
||||
});
|
||||
|
||||
it('returns a completion and cache usage from kimi-k3.1-pro', { timeout: INTEGRATION_TEST_TIMEOUT_MS }, async () => {
|
||||
const provider = new MoonshotProvider(
|
||||
{ apiKey: optionalEnv(ENV_VAR)! },
|
||||
makeMeteringStub(),
|
||||
);
|
||||
|
||||
const result = (await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'kimi-k3.1-pro',
|
||||
messages: [
|
||||
{ role: 'user', content: 'Say hi in one word.' },
|
||||
],
|
||||
// Reasoning counts against the cap, so leave room for it.
|
||||
max_tokens: 2048,
|
||||
reasoning_effort: 'low',
|
||||
}),
|
||||
)) as {
|
||||
message?: { content?: string };
|
||||
usage?: Record<string, number>;
|
||||
};
|
||||
|
||||
const text = result.message?.content;
|
||||
expect(typeof text === 'string' && text.length > 0).toBe(true);
|
||||
expect(result.usage).toMatchObject({
|
||||
prompt_tokens: expect.any(Number),
|
||||
completion_tokens: expect.any(Number),
|
||||
cached_tokens: expect.any(Number),
|
||||
cache_write_tokens: expect.any(Number),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -20,17 +20,15 @@
|
||||
/**
|
||||
* Offline unit tests for MoonshotProvider.
|
||||
*
|
||||
* Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock
|
||||
* redis) and constructs MoonshotProvider directly against the live
|
||||
* wired `MeteringService` so the recording side is exercised end-to-
|
||||
* end. Moonshot is OpenAI-compatible so the OpenAI SDK is mocked at
|
||||
* the module boundary; that's the real network egress point. Image-
|
||||
* inlining behaviour is covered by `utils/inlineImages.test.ts`; here we
|
||||
* stub `inlineHttpImageUrls` so http URLs in vision messages don't
|
||||
* trigger network fetches and only verify the provider invokes it
|
||||
* for vision-capable models. The companion integration test
|
||||
* (MoonshotProvider.integration.test.ts) exercises the real Moonshot
|
||||
* endpoint.
|
||||
* Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock redis) and
|
||||
* constructs MoonshotProvider directly against the live wired `MeteringService`
|
||||
* so the recording side is exercised end-to- end. Moonshot is OpenAI-compatible
|
||||
* so the OpenAI SDK is mocked at the module boundary; that's the real network
|
||||
* egress point. Image- inlining behaviour is covered by
|
||||
* `utils/inlineImages.test.ts`; here we stub `inlineHttpImageUrls` so http URLs
|
||||
* in vision messages don't trigger network fetches and only verify the provider
|
||||
* invokes it for vision-capable models. The companion integration test
|
||||
* (MoonshotProvider.integration.test.ts) exercises the real Moonshot endpoint.
|
||||
*/
|
||||
|
||||
import { Writable } from 'node:stream';
|
||||
@@ -46,14 +44,23 @@ import {
|
||||
type MockInstance,
|
||||
} from 'vitest';
|
||||
|
||||
import { ChatCompletionDriver } from '../../ChatCompletionDriver.js';
|
||||
import { clearUnhealthyRoutes } from '../../utils/providerHealth.js';
|
||||
import { SYSTEM_ACTOR } from '../../../../core/actor.js';
|
||||
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../../integrationTestUtil.js';
|
||||
import {
|
||||
readStreamResult,
|
||||
withTestActor,
|
||||
} from '../../../integrationTestUtil.js';
|
||||
import { AIChatStream } from '../../utils/Streaming.js';
|
||||
import { MOONSHOT_MODELS } from './models.js';
|
||||
import { MoonshotProvider } from './MoonshotProvider.js';
|
||||
import {
|
||||
estimatePromptTokens,
|
||||
estimateTextTokens,
|
||||
} from '../../utils/usageEstimate.js';
|
||||
import { modelLookupNames } from '../../utils/modelRouting.js';
|
||||
|
||||
// ── OpenAI SDK mock ─────────────────────────────────────────────────
|
||||
@@ -93,10 +100,23 @@ vi.mock('../../utils/inlineImages.js', () => ({
|
||||
// ── Test harness ────────────────────────────────────────────────────
|
||||
|
||||
let server: PuterServer;
|
||||
let driver: ChatCompletionDriver;
|
||||
let recordSpy: MockInstance<MeteringService['utilRecordUsageObject']>;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
driver = new ChatCompletionDriver(
|
||||
{
|
||||
providers: {
|
||||
moonshot: { apiKey: 'test-key' },
|
||||
ollama: { enabled: false },
|
||||
},
|
||||
} as never,
|
||||
server.clients,
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
await driver.onServerStart();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -140,6 +160,7 @@ const makeCapturingChatStream = () => {
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
clearUnhealthyRoutes();
|
||||
createMock.mockReset();
|
||||
openAICtor.mockReset();
|
||||
inlineHttpImageUrlsMock.mockReset();
|
||||
@@ -390,7 +411,10 @@ describe('MoonshotProvider image inlining', () => {
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'kimi-k3', // kimi-k3 declares image input modality
|
||||
messages: messages as unknown as { role: string; content: unknown }[],
|
||||
messages: messages as unknown as {
|
||||
role: string;
|
||||
content: unknown;
|
||||
}[],
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -782,3 +806,656 @@ describe('MoonshotProvider.checkModeration', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Kimi K3.1 Pro', () => {
|
||||
const models = ['kimi-k3.1-pro', 'kimi-k3.1-pro-highspeed'];
|
||||
/** Catalog rates in µ¢ per token: $2/$10 for Pro, $4/$20 for Highspeed. */
|
||||
const rates = (model: string) =>
|
||||
model.endsWith('highspeed')
|
||||
? { input: 400, output: 2000 }
|
||||
: { input: 200, output: 1000 };
|
||||
const usage = {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 10,
|
||||
prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 50 },
|
||||
};
|
||||
const response = () => ({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: 'answer',
|
||||
reasoning_content: 'reasoning',
|
||||
},
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: structuredClone(usage),
|
||||
});
|
||||
const fund = (balance: number) =>
|
||||
vi.spyOn(
|
||||
server.services.metering,
|
||||
'getUsageHeadroom',
|
||||
).mockResolvedValue({ balance, held: 0 });
|
||||
const user = (content: string) => [{ role: 'user', content }];
|
||||
|
||||
it.each(models)(
|
||||
'maps %s caps and reasoning without sampling parameters',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model,
|
||||
messages: user('hi'),
|
||||
max_tokens: 512,
|
||||
reasoning_effort: 'high',
|
||||
temperature: 0.2,
|
||||
top_p: 0.1,
|
||||
}),
|
||||
);
|
||||
const sent = createMock.mock.calls[0]![0];
|
||||
expect(sent).toMatchObject({
|
||||
model,
|
||||
max_completion_tokens: 512,
|
||||
reasoning_effort: 'high',
|
||||
});
|
||||
expect(sent).not.toHaveProperty('max_tokens');
|
||||
expect(sent).not.toHaveProperty('temperature');
|
||||
expect(sent).not.toHaveProperty('top_p');
|
||||
},
|
||||
);
|
||||
|
||||
it('uses the upstream default output cap and leaves thinking to the model', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
provider.complete({ model: models[0]!, messages: user('hi') }),
|
||||
);
|
||||
const sent = createMock.mock.calls[0]![0];
|
||||
expect(sent.max_completion_tokens).toBe(131072);
|
||||
expect(sent).not.toHaveProperty('thinking');
|
||||
expect(sent).not.toHaveProperty('reasoning_effort');
|
||||
});
|
||||
|
||||
it.each(['low', 'max'] as const)(
|
||||
'forwards %s reasoning effort',
|
||||
async (effort) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: models[0]!,
|
||||
messages: user('hi'),
|
||||
reasoning_effort: effort,
|
||||
}),
|
||||
);
|
||||
expect(createMock.mock.calls[0]![0].reasoning_effort).toBe(effort);
|
||||
},
|
||||
);
|
||||
|
||||
it('rejects an unsupported reasoning effort before inference', async () => {
|
||||
const { provider } = makeProvider();
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: models[0]!,
|
||||
messages: user('hi'),
|
||||
reasoning_effort: 'medium',
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ options: { reasoning: { effort: 'low' } }, expected: 'low' },
|
||||
{ options: { reasoning: { effort: 'max' } }, expected: 'max' },
|
||||
{
|
||||
options: { reasoning: { effort: 'low' }, reasoning_effort: 'high' },
|
||||
expected: 'high',
|
||||
},
|
||||
{ options: {}, expected: undefined },
|
||||
])(
|
||||
'sends the nested reasoning form through the driver: $options',
|
||||
async ({ options, expected }) => {
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: models[0]!,
|
||||
provider: 'moonshotai',
|
||||
messages: user('hi'),
|
||||
...options,
|
||||
} as never),
|
||||
);
|
||||
expect(createMock).toHaveBeenCalledTimes(1);
|
||||
expect(createMock.mock.calls[0]![0].reasoning_effort).toBe(
|
||||
expected,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['medium', 'none'])(
|
||||
'rejects the nested reasoning form %s through the driver',
|
||||
async (effort) => {
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
driver.complete({
|
||||
model: models[0]!,
|
||||
provider: 'moonshotai',
|
||||
messages: user('hi'),
|
||||
reasoning: { effort },
|
||||
} as never),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps K3 on max_tokens and bills its cache writes at the K3 write rate', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'kimi-k3',
|
||||
messages: user('hi'),
|
||||
max_tokens: 512,
|
||||
}),
|
||||
);
|
||||
const sent = createMock.mock.calls[0]![0];
|
||||
expect(sent).toMatchObject({ model: 'kimi-k3', max_tokens: 512 });
|
||||
expect(sent).not.toHaveProperty('max_completion_tokens');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
{
|
||||
prompt_tokens: 30,
|
||||
completion_tokens: 10,
|
||||
cached_tokens: 20,
|
||||
cache_write_tokens: 50,
|
||||
},
|
||||
expect.anything(),
|
||||
'moonshotai:kimi-k3',
|
||||
{
|
||||
prompt_tokens: 30 * 300,
|
||||
completion_tokens: 10 * 1500,
|
||||
cached_tokens: 20 * 30,
|
||||
cache_write_tokens: 50 * 300,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it.each(models)(
|
||||
'meters cache reads, writes and uncached input separately for %s',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
const result = await withTestActor(() =>
|
||||
provider.complete({ model, messages: user('hi') }),
|
||||
);
|
||||
const { input, output } = rates(model);
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
{
|
||||
prompt_tokens: 30,
|
||||
completion_tokens: 10,
|
||||
cached_tokens: 20,
|
||||
cache_write_tokens: 50,
|
||||
},
|
||||
expect.anything(),
|
||||
`moonshotai:${model}`,
|
||||
{
|
||||
prompt_tokens: 30 * input,
|
||||
completion_tokens: 10 * output,
|
||||
cached_tokens: 20 * 20,
|
||||
cache_write_tokens: 50 * input,
|
||||
},
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
usageDetails: {
|
||||
inputTokens: 30,
|
||||
outputTokens: 10,
|
||||
cacheReadTokens: 20,
|
||||
cacheWrite5mTokens: 50,
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('preserves assistant reasoning, text and tool-call order on replay', async () => {
|
||||
const { provider } = makeProvider();
|
||||
const toolCalls = ['a', 'b'].map((id) => ({
|
||||
id,
|
||||
type: 'function',
|
||||
function: { name: 'lookup', arguments: '{}' },
|
||||
}));
|
||||
const first = response();
|
||||
Object.assign(first.choices[0]!.message, { tool_calls: toolCalls });
|
||||
createMock.mockResolvedValueOnce(first);
|
||||
const result = await withTestActor(() =>
|
||||
provider.complete({ model: models[0]!, messages: user('hi') }),
|
||||
);
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: models[0]!,
|
||||
messages: [(result as { message: unknown }).message],
|
||||
}),
|
||||
);
|
||||
const replayed = createMock.mock.calls[1]![0].messages[0];
|
||||
expect(replayed).toMatchObject({
|
||||
role: 'assistant',
|
||||
content: 'answer',
|
||||
reasoning_content: 'reasoning',
|
||||
});
|
||||
expect(replayed).not.toHaveProperty('reasoning');
|
||||
expect(replayed.tool_calls).toEqual(toolCalls);
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: models[0]!,
|
||||
messages: [
|
||||
{
|
||||
role: 'assistant',
|
||||
reasoning: 'reasoning',
|
||||
content: [
|
||||
{ type: 'text', text: 'answer' },
|
||||
...['a', 'b'].map((id) => ({
|
||||
type: 'tool_use',
|
||||
id,
|
||||
name: 'lookup',
|
||||
input: {},
|
||||
})),
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(createMock.mock.calls[2]![0].messages[0]).toEqual({
|
||||
role: 'assistant',
|
||||
reasoning_content: 'reasoning',
|
||||
content: [{ type: 'text', text: 'answer' }],
|
||||
tool_calls: toolCalls,
|
||||
});
|
||||
});
|
||||
|
||||
it('meters a stream from its final usage chunk', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(
|
||||
asAsyncIterable([
|
||||
{ choices: [{ delta: { reasoning_content: 'reasoning' } }] },
|
||||
{ choices: [{ delta: { content: 'answer' } }] },
|
||||
{ choices: [], usage: structuredClone(usage) },
|
||||
]),
|
||||
);
|
||||
const result = await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: models[0]!,
|
||||
messages: user('hi'),
|
||||
stream: true,
|
||||
}),
|
||||
);
|
||||
const harness = makeCapturingChatStream();
|
||||
await (
|
||||
result as {
|
||||
init_chat_stream: (p: { chatStream: unknown }) => Promise<void>;
|
||||
}
|
||||
).init_chat_stream({ chatStream: harness.chatStream });
|
||||
expect(createMock.mock.calls[0]![0].stream_options).toEqual({
|
||||
include_usage: true,
|
||||
});
|
||||
expect(recordSpy).toHaveBeenCalledTimes(1);
|
||||
expect(recordSpy.mock.calls[0]![0]).toEqual({
|
||||
prompt_tokens: 30,
|
||||
completion_tokens: 10,
|
||||
cached_tokens: 20,
|
||||
cache_write_tokens: 50,
|
||||
});
|
||||
});
|
||||
|
||||
it.each(models)(
|
||||
'reserves the prompt at the input rate for %s',
|
||||
async (model) => {
|
||||
const { input, output } = rates(model);
|
||||
fund(2_000_000_000);
|
||||
const reserve = vi.spyOn(
|
||||
server.services.metering,
|
||||
'reserveCredits',
|
||||
);
|
||||
createMock.mockResolvedValueOnce({
|
||||
choices: [
|
||||
{
|
||||
message: { role: 'assistant', content: 'ok' },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
prompt_tokens: 1_000_000,
|
||||
completion_tokens: 10,
|
||||
prompt_tokens_details: {
|
||||
cached_tokens: 100_000,
|
||||
cache_write_tokens: 900_000,
|
||||
},
|
||||
},
|
||||
});
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
provider: 'moonshotai',
|
||||
messages: user('x'.repeat(8_000_000)),
|
||||
max_tokens: 10,
|
||||
}),
|
||||
);
|
||||
expect(reserve).toHaveBeenCalledTimes(1);
|
||||
expect(reserve.mock.calls[0]![1]).toBeGreaterThanOrEqual(
|
||||
1_000_000 * input + 10 * output,
|
||||
);
|
||||
expect(reserve.mock.calls[0]![1]).toBeLessThan(
|
||||
1_000_001 * input + 10 * output,
|
||||
);
|
||||
expect(recordSpy.mock.calls[0]![0]).toMatchObject({
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 10,
|
||||
cached_tokens: 100_000,
|
||||
cache_write_tokens: 900_000,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['kimi-k3', 'kimi-k2.6'])(
|
||||
'keeps the existing reservation for %s',
|
||||
async (model) => {
|
||||
const entry = MOONSHOT_MODELS.find(
|
||||
(candidate) => candidate.id === model,
|
||||
)!;
|
||||
fund(2_000_000_000);
|
||||
const reserve = vi.spyOn(
|
||||
server.services.metering,
|
||||
'reserveCredits',
|
||||
);
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
provider: 'moonshotai',
|
||||
messages: [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'x'.repeat(8000),
|
||||
reasoning: 'thought '.repeat(90000),
|
||||
},
|
||||
],
|
||||
max_tokens: 10,
|
||||
}),
|
||||
);
|
||||
expect(reserve.mock.calls[0]![1]).toBe(
|
||||
1000 * Number(entry.costs.prompt_tokens) +
|
||||
10 * Number(entry.costs.completion_tokens),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(models)(
|
||||
'rejects an underfunded request for %s before inference',
|
||||
async (model) => {
|
||||
// Short of the prompt alone at the input rate.
|
||||
fund(model.endsWith('highspeed') ? 300_000_000 : 150_000_000);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
provider: 'moonshotai',
|
||||
messages: user('x'.repeat(8_000_000)),
|
||||
max_tokens: 10,
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 402 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([null, ''])(
|
||||
'round-trips tool-only assistant content %s through the driver',
|
||||
async (content) => {
|
||||
const toolCalls = [
|
||||
{
|
||||
id: 'call_paris',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'weather',
|
||||
arguments: '{"city":"Paris"}',
|
||||
},
|
||||
},
|
||||
];
|
||||
createMock.mockResolvedValueOnce({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content,
|
||||
reasoning_content: 'Check weather',
|
||||
tool_calls: toolCalls,
|
||||
},
|
||||
finish_reason: 'tool_calls',
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1 },
|
||||
});
|
||||
const first = await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: models[0]!,
|
||||
provider: 'moonshotai',
|
||||
messages: user('weather?'),
|
||||
}),
|
||||
);
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: models[0]!,
|
||||
provider: 'moonshotai',
|
||||
messages: [
|
||||
(first as { message: unknown }).message,
|
||||
{
|
||||
role: 'tool',
|
||||
tool_call_id: 'call_paris',
|
||||
content: 'sunny',
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const assistant = createMock.mock.calls[1]![0].messages.find(
|
||||
(message: { role: string }) => message.role === 'assistant',
|
||||
);
|
||||
expect(assistant).toMatchObject({
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
reasoning_content: 'Check weather',
|
||||
});
|
||||
expect(assistant).not.toHaveProperty('normalized');
|
||||
expect(assistant.tool_calls).toEqual(toolCalls);
|
||||
expect(
|
||||
JSON.parse(assistant.tool_calls[0].function.arguments),
|
||||
).toEqual({ city: 'Paris' });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([{ city: 'Paris' }, '{"city":"Paris"}'])(
|
||||
'keeps native tool_use input %j through the driver',
|
||||
async (input) => {
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: models[0]!,
|
||||
provider: 'moonshotai',
|
||||
messages: [
|
||||
{
|
||||
role: 'assistant',
|
||||
reasoning_content: 'Check weather',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'call_paris',
|
||||
name: 'weather',
|
||||
input,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: 'tool',
|
||||
tool_call_id: 'call_paris',
|
||||
content: 'sunny',
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const assistant = createMock.mock.calls[0]![0].messages.find(
|
||||
(message: { role: string }) => message.role === 'assistant',
|
||||
);
|
||||
expect(assistant.reasoning_content).toBe('Check weather');
|
||||
expect(assistant.tool_calls[0].function.arguments).toBe(
|
||||
'{"city":"Paris"}',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(
|
||||
models.flatMap((model) =>
|
||||
['reasoning', 'reasoning_content', 'both'].map((alias) => ({
|
||||
model,
|
||||
alias,
|
||||
})),
|
||||
),
|
||||
)(
|
||||
'reserves replayed reasoning once with wire precedence: $model $alias',
|
||||
async ({ model, alias }) => {
|
||||
const text = 'reasoning '.repeat(90000);
|
||||
const assistant = {
|
||||
role: 'assistant',
|
||||
content: 'answer',
|
||||
...(alias !== 'reasoning_content'
|
||||
? { reasoning: alias === 'both' ? 'short' : text }
|
||||
: {}),
|
||||
...(alias !== 'reasoning' ? { reasoning_content: text } : {}),
|
||||
};
|
||||
const messages = [
|
||||
...user('hi'),
|
||||
assistant,
|
||||
...user('continue'),
|
||||
];
|
||||
const tokens =
|
||||
estimatePromptTokens(messages) + estimateTextTokens(text);
|
||||
const { input, output } = rates(model);
|
||||
const headroom = fund(2_000_000_000);
|
||||
const reserve = vi.spyOn(
|
||||
server.services.metering,
|
||||
'reserveCredits',
|
||||
);
|
||||
createMock.mockResolvedValueOnce(response());
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
messages: structuredClone(messages),
|
||||
max_tokens: 10,
|
||||
}),
|
||||
);
|
||||
expect(reserve.mock.calls[0]![1]).toBe(
|
||||
tokens * input + 10 * output,
|
||||
);
|
||||
expect(
|
||||
createMock.mock.calls[0]![0].messages[1].reasoning_content,
|
||||
).toBe(text);
|
||||
headroom.mockResolvedValue({
|
||||
balance: tokens * input - 1,
|
||||
held: 0,
|
||||
});
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
messages: structuredClone(messages),
|
||||
max_tokens: 10,
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 402 });
|
||||
expect(createMock).toHaveBeenCalledTimes(1);
|
||||
expect(reserve).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(models)(
|
||||
'charges the estimate, replayed reasoning included, when a %s stream is interrupted',
|
||||
async (model) => {
|
||||
const reasoning = 'history '.repeat(1000);
|
||||
const messages = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'answer',
|
||||
reasoning: 'ignored alias',
|
||||
reasoning_content: reasoning,
|
||||
},
|
||||
...user('hi'),
|
||||
];
|
||||
const expectedInput =
|
||||
estimatePromptTokens(messages) + estimateTextTokens(reasoning);
|
||||
createMock.mockResolvedValueOnce(
|
||||
(async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: 'partial answer' } }],
|
||||
};
|
||||
throw new Error('upstream connection lost');
|
||||
})(),
|
||||
);
|
||||
const result = await withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
messages: structuredClone(messages),
|
||||
stream: true,
|
||||
}),
|
||||
);
|
||||
const text = (await readStreamResult(result)).toString();
|
||||
expect(text).toContain('upstream connection lost');
|
||||
await vi.waitFor(() => expect(recordSpy).toHaveBeenCalledTimes(1));
|
||||
expect(
|
||||
createMock.mock.calls[0]![0].messages[0].reasoning_content,
|
||||
).toBe(reasoning);
|
||||
expect(recordSpy.mock.calls[0]![0]).toHaveProperty(
|
||||
'estimated_prompt_tokens',
|
||||
expectedInput,
|
||||
);
|
||||
expect(recordSpy.mock.calls[0]![0]).toHaveProperty(
|
||||
'estimated_completion_tokens',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(models)(
|
||||
'charges the estimate when a %s stream completes without a usage chunk',
|
||||
async (model) => {
|
||||
createMock.mockResolvedValueOnce(
|
||||
asAsyncIterable([
|
||||
{ choices: [{ delta: { content: 'answer' } }] },
|
||||
{ choices: [{ delta: {}, finish_reason: 'stop' }] },
|
||||
]),
|
||||
);
|
||||
const result = await withTestActor(() =>
|
||||
driver.complete({
|
||||
model,
|
||||
provider: 'moonshotai',
|
||||
messages: user('hi'),
|
||||
stream: true,
|
||||
}),
|
||||
);
|
||||
const events = (await readStreamResult(result))
|
||||
.toString()
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => JSON.parse(line));
|
||||
expect(events.some((event) => event.type === 'error')).toBe(false);
|
||||
await vi.waitFor(() => expect(recordSpy).toHaveBeenCalledTimes(1));
|
||||
expect(recordSpy.mock.calls[0]![0]).toMatchObject({
|
||||
estimated_prompt_tokens: expect.any(Number),
|
||||
estimated_completion_tokens: expect.any(Number),
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -18,14 +18,39 @@
|
||||
*/
|
||||
|
||||
import { OpenAI } from 'openai';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
|
||||
import type {
|
||||
IChatModel,
|
||||
ICompleteArguments,
|
||||
PuterMessage,
|
||||
} from '../../types.js';
|
||||
import { openAICompatParams } from '../../utils/openaiParams.js';
|
||||
import { sdkClientOptions } from '../../utils/sdkClient.js';
|
||||
import { estimateTextTokens } from '../../utils/usageEstimate.js';
|
||||
import {
|
||||
type ChatProviderConfig,
|
||||
OpenAICompatProvider,
|
||||
} from '../OpenAICompatProvider.js';
|
||||
import { MOONSHOT_MODELS } from './models.js';
|
||||
|
||||
/**
|
||||
* The K3-series request shape: reasoning always on, capped by
|
||||
* `max_completion_tokens`.
|
||||
*/
|
||||
const K31_MODEL_IDS = new Set(['kimi-k3.1-pro', 'kimi-k3.1-pro-highspeed']);
|
||||
const K31_REASONING_EFFORTS = ['low', 'high', 'max'];
|
||||
/** Moonshot's own output cap when the request names none. */
|
||||
const K31_DEFAULT_MAX_COMPLETION_TOKENS = 131_072;
|
||||
|
||||
const isK31 = (model: IChatModel) => K31_MODEL_IDS.has(model.id);
|
||||
|
||||
/** A replayed assistant turn's reasoning, under the wire key or Puter's. */
|
||||
const replayedReasoning = (message: PuterMessage): string | undefined => {
|
||||
const reasoning = message.reasoning_content ?? message.reasoning;
|
||||
return typeof reasoning === 'string' ? reasoning : undefined;
|
||||
};
|
||||
|
||||
export class MoonshotProvider extends OpenAICompatProvider {
|
||||
constructor(config: ChatProviderConfig, meteringService: MeteringService) {
|
||||
super(meteringService, {
|
||||
@@ -41,4 +66,79 @@ export class MoonshotProvider extends OpenAICompatProvider {
|
||||
inlineImages: 'vision',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* K3.1 Pro reads replayed reasoning back as prompt, which the message
|
||||
* estimate the credit gate works from doesn't count.
|
||||
*/
|
||||
requestPricing(args: ICompleteArguments, model: IChatModel) {
|
||||
if (!isK31(model)) return {};
|
||||
const additionalPromptTokens = (args.messages ?? []).reduce(
|
||||
(tokens, message) => {
|
||||
if (message.role !== 'assistant') return tokens;
|
||||
const reasoning = replayedReasoning(message);
|
||||
return tokens + (reasoning ? estimateTextTokens(reasoning) : 0);
|
||||
},
|
||||
0,
|
||||
);
|
||||
return { additionalPromptTokens };
|
||||
}
|
||||
|
||||
protected override preservesToolCallOrder(model: IChatModel) {
|
||||
return isK31(model);
|
||||
}
|
||||
|
||||
protected override preservesAssistantContent(model: IChatModel) {
|
||||
return isK31(model);
|
||||
}
|
||||
|
||||
/** K3.1 Pro wants a turn's reasoning back under `reasoning_content`, once. */
|
||||
protected override prepareMessages(
|
||||
messages: PuterMessage[],
|
||||
model: IChatModel,
|
||||
) {
|
||||
if (!isK31(model)) return messages;
|
||||
return messages.map((message) => {
|
||||
if (message.role !== 'assistant') return message;
|
||||
const { reasoning, normalized: _normalized, ...rest } = message;
|
||||
const replayed = replayedReasoning(message);
|
||||
return replayed === undefined
|
||||
? rest
|
||||
: { ...rest, reasoning_content: replayed };
|
||||
});
|
||||
}
|
||||
|
||||
protected override vendorParams(
|
||||
params: Record<string, unknown>,
|
||||
args: ICompleteArguments,
|
||||
model: IChatModel,
|
||||
) {
|
||||
if (!isK31(model)) return params;
|
||||
const effort = args.reasoning_effort ?? args.reasoning?.effort;
|
||||
if (effort !== undefined && !K31_REASONING_EFFORTS.includes(effort)) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`reasoning_effort must be one of ${K31_REASONING_EFFORTS.join(', ')} for ${model.id}`,
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
// The K3 series deprecates `max_tokens` in favour of `max_completion_tokens`.
|
||||
const { max_tokens, ...rest } = params;
|
||||
return {
|
||||
...rest,
|
||||
...openAICompatParams(
|
||||
{ ...args, reasoning_effort: effort },
|
||||
'chat',
|
||||
{
|
||||
only: [
|
||||
'tool_choice',
|
||||
'response_format',
|
||||
'reasoning_effort',
|
||||
],
|
||||
},
|
||||
),
|
||||
max_completion_tokens:
|
||||
max_tokens ?? K31_DEFAULT_MAX_COMPLETION_TOKENS,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,53 @@
|
||||
import type { IChatModel } from '../../types.js';
|
||||
|
||||
export const MOONSHOT_MODELS: IChatModel[] = [
|
||||
// -- Kimi K3.1 Pro ------------------------------------------------
|
||||
{
|
||||
puterId: 'moonshotai:moonshotai/kimi-k3.1-pro',
|
||||
id: 'kimi-k3.1-pro',
|
||||
name: 'Kimi K3.1 Pro',
|
||||
aliases: ['moonshotai/kimi-k3.1-pro', 'moonshot/kimi-k3.1-pro'],
|
||||
modalities: { input: ['text', 'image', 'video'], output: ['text'] },
|
||||
costs_currency: 'usd-cents',
|
||||
input_cost_key: 'prompt_tokens',
|
||||
output_cost_key: 'completion_tokens',
|
||||
costs: {
|
||||
tokens: 1_000_000,
|
||||
prompt_tokens: 200, // $2.00 per 1M
|
||||
completion_tokens: 1000, // $10.00 per 1M
|
||||
cached_tokens: 20, // $0.20 per 1M
|
||||
cache_write_tokens: 200, // $2.00 per 1M
|
||||
},
|
||||
context: 1_048_576,
|
||||
max_tokens: 1_048_576,
|
||||
tool_call: true,
|
||||
release_date: '2026-10-24',
|
||||
},
|
||||
{
|
||||
puterId: 'moonshotai:moonshotai/kimi-k3.1-pro-highspeed',
|
||||
id: 'kimi-k3.1-pro-highspeed',
|
||||
name: 'Kimi K3.1 Pro Highspeed',
|
||||
aliases: [
|
||||
'moonshotai/kimi-k3.1-pro-highspeed',
|
||||
'moonshot/kimi-k3.1-pro-highspeed',
|
||||
],
|
||||
modalities: { input: ['text', 'image', 'video'], output: ['text'] },
|
||||
costs_currency: 'usd-cents',
|
||||
input_cost_key: 'prompt_tokens',
|
||||
output_cost_key: 'completion_tokens',
|
||||
costs: {
|
||||
tokens: 1_000_000,
|
||||
prompt_tokens: 400, // $4.00 per 1M
|
||||
completion_tokens: 2000, // $20.00 per 1M
|
||||
cached_tokens: 20, // $0.20 per 1M
|
||||
cache_write_tokens: 400, // $4.00 per 1M
|
||||
},
|
||||
context: 1_048_576,
|
||||
max_tokens: 1_048_576,
|
||||
tool_call: true,
|
||||
release_date: '2026-10-24',
|
||||
},
|
||||
|
||||
// -- Flagship ----------------------------------------------------
|
||||
{
|
||||
puterId: 'moonshotai:moonshotai/kimi-k3',
|
||||
@@ -35,6 +82,7 @@ export const MOONSHOT_MODELS: IChatModel[] = [
|
||||
prompt_tokens: 300, // $3.00 per 1M
|
||||
completion_tokens: 1500, // $15.00 per 1M
|
||||
cached_tokens: 30, // $0.30 per 1M
|
||||
cache_write_tokens: 300, // $3.00 per 1M
|
||||
},
|
||||
context: 1_048_576,
|
||||
max_tokens: 1_048_576,
|
||||
|
||||
+26
-22
@@ -177,6 +177,7 @@ describe('OpenAiChatProvider model catalog', () => {
|
||||
expect(ids).toContain('gpt-5.2-2025-12-11');
|
||||
expect(ids).toContain('gpt-6-astra');
|
||||
expect(ids).toContain('gpt-6.1-sol');
|
||||
expect(ids).toContain('gpt-6.1-terra');
|
||||
expect(ids).toContain('gpt-6-sol');
|
||||
expect(ids).toContain('gpt-6-luna');
|
||||
});
|
||||
@@ -348,30 +349,33 @@ describe('OpenAiChatProvider.complete request shape', () => {
|
||||
expect(args.safety_identifier).toBe('puter-u42');
|
||||
});
|
||||
|
||||
it.each(['gpt-6-astra', 'gpt-6.1-sol', 'gpt-6-sol', 'gpt-6-luna'])(
|
||||
'resolves the namespaced %s alias',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
it.each([
|
||||
'gpt-6-astra',
|
||||
'gpt-6.1-sol',
|
||||
'gpt-6.1-terra',
|
||||
'gpt-6-sol',
|
||||
'gpt-6-luna',
|
||||
])('resolves the namespaced %s alias', async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: `openai/${model}`,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
reasoning_effort: 'low',
|
||||
}),
|
||||
);
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: `openai/${model}`,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
reasoning_effort: 'low',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(model);
|
||||
expect(createMock.mock.calls[0]![0].reasoning_effort).toBe('low');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
`openai:${model}`,
|
||||
expect.any(Object),
|
||||
);
|
||||
},
|
||||
);
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(model);
|
||||
expect(createMock.mock.calls[0]![0].reasoning_effort).toBe('low');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
`openai:${model}`,
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('omits parallel_tool_calls/tool_choice when no tool is sent (Chat Completions rejects them alone)', async () => {
|
||||
const { provider } = makeProvider();
|
||||
|
||||
@@ -224,6 +224,7 @@ describe('OpenAiResponsesChatProvider model catalog', () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
['gpt-6.1-terra', '2026-06-01', '2026-10-26', 100, 5, 500],
|
||||
['gpt-6.1-sol', '2026-04-30', '2026-09-29', 200, 10, 1000],
|
||||
['gpt-6-sol', '2026-04-20', '2026-09-22', 200, 20, 1000],
|
||||
['gpt-6-luna', '2026-05-18', '2026-09-22', 10, 1, 50],
|
||||
@@ -244,6 +245,7 @@ describe('OpenAiResponsesChatProvider model catalog', () => {
|
||||
tokens: 1_000_000,
|
||||
prompt_tokens: input,
|
||||
cached_tokens: cached,
|
||||
cache_write_tokens: input * 1.25,
|
||||
completion_tokens: output,
|
||||
},
|
||||
context: 1_050_000,
|
||||
@@ -332,6 +334,7 @@ describe('OpenAiResponsesChatProvider.complete request shape', () => {
|
||||
'gpt-6-luna',
|
||||
'gpt-6-astra',
|
||||
'gpt-6.1-sol',
|
||||
'gpt-6.1-terra',
|
||||
'gpt-5.6',
|
||||
'openai/gpt-5.6',
|
||||
'openai/gpt-5.6-luna',
|
||||
@@ -367,7 +370,7 @@ describe('OpenAiResponsesChatProvider.complete request shape', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['gpt-5.6-luna', 'gpt-6-luna'])(
|
||||
it.each(['gpt-5.6-luna', 'gpt-6-luna', 'gpt-6.1-terra'])(
|
||||
'preserves sampling with reasoning disabled for %s',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
@@ -620,7 +623,13 @@ describe('OpenAiResponsesChatProvider.complete request shape', () => {
|
||||
expect(gpt41Args).not.toHaveProperty('verbosity');
|
||||
});
|
||||
|
||||
it.each(['gpt-6-astra', 'gpt-6.1-sol', 'gpt-6-sol', 'gpt-6-luna'])(
|
||||
it.each([
|
||||
'gpt-6-astra',
|
||||
'gpt-6.1-sol',
|
||||
'gpt-6.1-terra',
|
||||
'gpt-6-sol',
|
||||
'gpt-6-luna',
|
||||
])(
|
||||
'maps flat controls to Responses options for %s aliases',
|
||||
async (model) => {
|
||||
const { provider } = makeProvider();
|
||||
@@ -860,47 +869,51 @@ describe('OpenAiResponsesChatProvider.complete non-stream output', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('bills the whole request at long-context rates past 272K input tokens', async () => {
|
||||
const sol = OPEN_AI_MODELS.find((m) => m.id === 'gpt-6-sol')!;
|
||||
const { provider } = makeProvider();
|
||||
responsesCreateMock.mockResolvedValueOnce({
|
||||
output: [{ role: 'assistant' }],
|
||||
output_text: 'hi',
|
||||
usage: {
|
||||
input_tokens: 300_000,
|
||||
output_tokens: 10_000,
|
||||
input_tokens_details: {
|
||||
cached_tokens: 50_000,
|
||||
cache_write_tokens: 20_000,
|
||||
it.each([
|
||||
['gpt-6-sol', 119],
|
||||
['gpt-6.1-terra', 59],
|
||||
] as const)(
|
||||
'bills %s at long-context rates past 272K input tokens',
|
||||
async (modelId, total) => {
|
||||
const model = OPEN_AI_MODELS.find((m) => m.id === modelId)!;
|
||||
const { provider } = makeProvider();
|
||||
responsesCreateMock.mockResolvedValueOnce({
|
||||
output: [{ role: 'assistant' }],
|
||||
output_text: 'hi',
|
||||
usage: {
|
||||
input_tokens: 300_000,
|
||||
output_tokens: 10_000,
|
||||
input_tokens_details: {
|
||||
cached_tokens: 50_000,
|
||||
cache_write_tokens: 20_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'gpt-6-sol',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: modelId,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const [, , , overrides] = recordSpy.mock.calls[0]!;
|
||||
// $0.92 + $0.02 + $0.10 + $0.15 = $1.19, against $0.62 at the
|
||||
// standard rates.
|
||||
expect(overrides).toEqual({
|
||||
prompt_tokens: 230_000 * Number(sol.costs.prompt_tokens) * 2,
|
||||
cached_tokens: 50_000 * Number(sol.costs.cached_tokens) * 2,
|
||||
cache_write_tokens:
|
||||
20_000 * Number(sol.costs.cache_write_tokens) * 2,
|
||||
completion_tokens:
|
||||
10_000 * Number(sol.costs.completion_tokens) * 1.5,
|
||||
});
|
||||
const totalCents =
|
||||
Object.values(overrides as Record<string, number>).reduce(
|
||||
(a, b) => a + b,
|
||||
0,
|
||||
) / 1_000_000;
|
||||
expect(totalCents).toBeCloseTo(119);
|
||||
});
|
||||
const [, , , overrides] = recordSpy.mock.calls[0]!;
|
||||
expect(overrides).toEqual({
|
||||
prompt_tokens: 230_000 * Number(model.costs.prompt_tokens) * 2,
|
||||
cached_tokens: 50_000 * Number(model.costs.cached_tokens) * 2,
|
||||
cache_write_tokens:
|
||||
20_000 * Number(model.costs.cache_write_tokens) * 2,
|
||||
completion_tokens:
|
||||
10_000 * Number(model.costs.completion_tokens) * 1.5,
|
||||
});
|
||||
const totalCents =
|
||||
Object.values(overrides as Record<string, number>).reduce(
|
||||
(a, b) => a + b,
|
||||
0,
|
||||
) / 1_000_000;
|
||||
expect(totalCents).toBeCloseTo(total);
|
||||
},
|
||||
);
|
||||
|
||||
it('bills cached tokens at the input rate when the model prices no cache read', async () => {
|
||||
// gpt-5.4-pro is responses-API-only and its catalogue entry has no
|
||||
|
||||
@@ -31,6 +31,31 @@ const GPT_LONG_CONTEXT_PRICING = {
|
||||
|
||||
// Hardcoded from https://models.dev/api.json
|
||||
export const OPEN_AI_MODELS: IChatModel[] = [
|
||||
{
|
||||
puterId: 'openai:openai/gpt-6.1-terra',
|
||||
id: 'gpt-6.1-terra',
|
||||
responsesSampling: 'reasoningDisabled',
|
||||
modalities: { input: ['text', 'image'], output: ['text'] },
|
||||
open_weights: false,
|
||||
tool_call: true,
|
||||
knowledge: '2026-06-01',
|
||||
release_date: '2026-10-26',
|
||||
aliases: ['openai/gpt-6.1-terra'],
|
||||
costs_currency: 'usd-cents',
|
||||
input_cost_key: 'prompt_tokens',
|
||||
output_cost_key: 'completion_tokens',
|
||||
costs: {
|
||||
tokens: 1_000_000,
|
||||
prompt_tokens: 100,
|
||||
cached_tokens: 5,
|
||||
cache_write_tokens: 100 * 1.25,
|
||||
completion_tokens: 500,
|
||||
},
|
||||
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
|
||||
context: 1_050_000,
|
||||
max_tokens: 128_000,
|
||||
responses_api: true,
|
||||
},
|
||||
{
|
||||
puterId: 'openai:openai/gpt-6.1-sol',
|
||||
id: 'gpt-6.1-sol',
|
||||
|
||||
@@ -238,7 +238,7 @@ export interface ICompleteArguments {
|
||||
'auto' | 'default' | 'flex' | 'scale' | 'priority' | undefined;
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
reasoning?: { effort: 'low' | 'medium' | 'high' } | undefined;
|
||||
reasoning?: { effort: ReasoningEffort } | undefined;
|
||||
text?: { verbosity?: 'low' | 'medium' | 'high' | undefined } | undefined;
|
||||
reasoning_effort?: ReasoningEffort | undefined;
|
||||
verbosity?: 'low' | 'medium' | 'high' | undefined;
|
||||
@@ -345,6 +345,8 @@ export interface IChatProvider {
|
||||
inputKey?: string;
|
||||
outputKey?: string;
|
||||
extraCost?: number;
|
||||
/** Provider-specific prompt tokens outside message.content. */
|
||||
additionalPromptTokens?: number;
|
||||
maxTokens?: number;
|
||||
};
|
||||
/**
|
||||
|
||||
@@ -170,7 +170,12 @@ const hoistToolUseBlocks = (
|
||||
type: 'function',
|
||||
function: {
|
||||
name: block.name,
|
||||
arguments: JSON.stringify(block.input),
|
||||
// A replayed OpenAI-shaped call arrives with `input` already
|
||||
// serialized; encoding it again sends a JSON string literal.
|
||||
arguments:
|
||||
typeof block.input === 'string'
|
||||
? block.input
|
||||
: JSON.stringify(block.input),
|
||||
},
|
||||
...(extraContent && block.extra_content
|
||||
? { extra_content: block.extra_content }
|
||||
@@ -190,14 +195,22 @@ const hoistToolUseBlocks = (
|
||||
* make changes in-place.
|
||||
*
|
||||
* @param {Message[]} messages - Array of normalized messages
|
||||
* @param {{ preserveToolCallOrder?: boolean; keepExtraContent?: boolean }} [opts]
|
||||
* @param {{
|
||||
* preserveToolCallOrder?: boolean;
|
||||
* keepExtraContent?: boolean;
|
||||
* keepAssistantContent?: boolean;
|
||||
* }} [opts]
|
||||
* `preserveToolCallOrder` keeps a turn's tool calls in the order the model made
|
||||
* them. `keepExtraContent` keeps each tool call's `extra_content`.
|
||||
* @returns {Message[]} - Array of messages in OpenAI format
|
||||
*/
|
||||
export const process_input_messages = async (
|
||||
messages,
|
||||
{ preserveToolCallOrder = false, keepExtraContent = false } = {},
|
||||
{
|
||||
preserveToolCallOrder = false,
|
||||
keepExtraContent = false,
|
||||
keepAssistantContent = false,
|
||||
} = {},
|
||||
) => {
|
||||
for (const msg of messages) {
|
||||
if (!msg.content) continue;
|
||||
@@ -220,7 +233,12 @@ export const process_input_messages = async (
|
||||
extraContent: keepExtraContent,
|
||||
})
|
||||
)
|
||||
msg.content = null;
|
||||
msg.content =
|
||||
keepAssistantContent &&
|
||||
msg.role === 'assistant' &&
|
||||
content.length
|
||||
? content
|
||||
: null;
|
||||
|
||||
// coerce tool results
|
||||
// (we assume multiple tool results were already split into separate messages)
|
||||
@@ -470,11 +488,22 @@ export const extractMeteredUsage = (usage) => {
|
||||
* @param {import('openai/resources/completions.mjs').CompletionUsage | undefined} usage
|
||||
*/
|
||||
export const splitCachedPrompt = (usage) => {
|
||||
const cachedTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0;
|
||||
const details = usage?.prompt_tokens_details;
|
||||
const cachedTokens = details?.cached_tokens ?? 0;
|
||||
// Moonshot counts the prompt tokens it wrote to its prefix cache here;
|
||||
// they're billed at the model's write rate, not the plain prompt rate.
|
||||
const writeTokens =
|
||||
typeof details?.cache_write_tokens === 'number'
|
||||
? details.cache_write_tokens
|
||||
: undefined;
|
||||
return {
|
||||
prompt_tokens: (usage?.prompt_tokens ?? 0) - cachedTokens,
|
||||
prompt_tokens:
|
||||
(usage?.prompt_tokens ?? 0) - cachedTokens - (writeTokens ?? 0),
|
||||
completion_tokens: usage?.completion_tokens ?? 0,
|
||||
cached_tokens: cachedTokens,
|
||||
...(writeTokens !== undefined
|
||||
? { cache_write_tokens: writeTokens }
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -331,6 +331,29 @@ describe('process_input_messages', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a replayed call's serialized arguments verbatim", async () => {
|
||||
// An OpenAI-shaped `tool_calls` entry normalizes with its arguments
|
||||
// still a JSON string.
|
||||
const messages: Array<Record<string, unknown>> = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'call_1',
|
||||
name: 'lookup',
|
||||
input: '{"q":"puter"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const [out] = (await process_input_messages(messages)) as Array<{
|
||||
tool_calls: Array<{ function: { arguments: string } }>;
|
||||
}>;
|
||||
expect(out!.tool_calls[0]!.function.arguments).toBe('{"q":"puter"}');
|
||||
});
|
||||
|
||||
describe('tool call extra_content', () => {
|
||||
const turn = (): Array<Record<string, unknown>> => [
|
||||
{
|
||||
@@ -772,6 +795,28 @@ describe('process_input_messages_responses_api', () => {
|
||||
expect(out!.content).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps a replayed call's serialized arguments verbatim", async () => {
|
||||
const messages: Array<Record<string, unknown>> = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'call_1',
|
||||
name: 'lookup',
|
||||
input: '{"q":"puter"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const [out] = (await process_input_messages_responses_api(
|
||||
messages,
|
||||
)) as Array<Record<string, unknown>>;
|
||||
expect(out!.type).toBe('function_call');
|
||||
expect(out!.arguments).toBe('{"q":"puter"}');
|
||||
});
|
||||
|
||||
it('rewrites tool_result blocks into function_call_output', async () => {
|
||||
const messages: Array<Record<string, unknown>> = [
|
||||
{
|
||||
@@ -888,6 +933,35 @@ describe('splitCachedPrompt', () => {
|
||||
cached_tokens: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('takes cache writes out of prompt_tokens under their own key', () => {
|
||||
expect(
|
||||
splitCachedPrompt({
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
prompt_tokens_details: { cached_tokens: 3, cache_write_tokens: 4 },
|
||||
}),
|
||||
).toEqual({
|
||||
prompt_tokens: 3,
|
||||
completion_tokens: 5,
|
||||
cached_tokens: 3,
|
||||
cache_write_tokens: 4,
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves cache writes out when the upstream reports no count', () => {
|
||||
expect(
|
||||
splitCachedPrompt({
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
prompt_tokens_details: { cached_tokens: 3, cache_write_tokens: null },
|
||||
}),
|
||||
).toEqual({
|
||||
prompt_tokens: 7,
|
||||
completion_tokens: 5,
|
||||
cached_tokens: 3,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── recorded usage costs ────────────────────────────────────────────
|
||||
|
||||
@@ -289,6 +289,9 @@ describe('clampReasoningEffort', () => {
|
||||
['gpt-5.5', 'max', 'xhigh'],
|
||||
['gpt-5', 'high', 'high'],
|
||||
['gpt-6-astra-1', 'none', 'low'],
|
||||
...(['none', 'low', 'medium', 'high', 'xhigh', 'max'] as const).map(
|
||||
(effort) => ['gpt-6.1-terra', effort, effort] as const,
|
||||
),
|
||||
] as const)(
|
||||
'clamps %s requesting %s to %s',
|
||||
(modelId, requested, expected) => {
|
||||
|
||||
@@ -339,13 +339,16 @@ export class ImageGenerationDriver extends PuterDriver {
|
||||
);
|
||||
|
||||
if (resolvedOutputPath) {
|
||||
await saveGeneratedMediaToFS(
|
||||
// False when write access was revoked while the job ran; the
|
||||
// result still comes back.
|
||||
const outputSaved = await saveGeneratedMediaToFS(
|
||||
this.services.fs,
|
||||
actor,
|
||||
result,
|
||||
resolvedOutputPath,
|
||||
{ noun: 'image', defaultType: 'application/octet-stream' },
|
||||
);
|
||||
Context.set('driverMetadata', { outputSaved });
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
type MockInstance,
|
||||
} from 'vitest';
|
||||
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
@@ -349,3 +350,55 @@ describe('KlingImageProvider billing', () => {
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// -- After the caller leaves -----------------------------------------
|
||||
|
||||
describe('KlingImageProvider after the caller leaves', () => {
|
||||
it('keeps polling after a disconnect and meters the finished image', async () => {
|
||||
mockFlow(envelope(task('processing')), envelope(succeeded));
|
||||
const url = await withTestActor(() => {
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return makeProvider().generate({ prompt: 'a red apple' });
|
||||
});
|
||||
expect(url).toBe('https://kling.example/1.png');
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('meters an image that finishes after the window', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(envelope(task('submitted')));
|
||||
const realNow = Date.now.bind(Date);
|
||||
let skew = 0;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => realNow() + skew);
|
||||
let finished = false;
|
||||
fetchSpy.mockImplementation(async () => {
|
||||
skew = 6 * 60 * 1000;
|
||||
return finished
|
||||
? envelope(succeeded)
|
||||
: envelope(task('processing'));
|
||||
});
|
||||
|
||||
let failure: unknown;
|
||||
try {
|
||||
await generate({ model: 'kling-v3-omni', quality: '4k' });
|
||||
} catch (e) {
|
||||
failure = e;
|
||||
}
|
||||
expect(failure).toMatchObject({
|
||||
statusCode: 504,
|
||||
legacyCode: 'upstream_timeout',
|
||||
message: 'Timed out waiting for Kling image generation to complete',
|
||||
});
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
|
||||
finished = true;
|
||||
await vi.waitFor(() =>
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1),
|
||||
);
|
||||
const [, usageType, amount, cost] = incrementUsageSpy.mock.calls[0]!;
|
||||
expect(usageType).toBe('kling-image-generation:kling-v3-omni:4k');
|
||||
expect(amount).toBe(1);
|
||||
expect(cost).toBe(5.6 * 1_000_000);
|
||||
});
|
||||
});
|
||||
@@ -24,9 +24,12 @@ import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import { klingImageInput, klingRequest } from '../../../util/kling.js';
|
||||
import {
|
||||
CONTENT_FILTER_PATTERN,
|
||||
isTransientUpstreamError,
|
||||
sanitizeUpstreamMessage,
|
||||
} from '../../../util/upstreamErrors.js';
|
||||
import {
|
||||
JobHold,
|
||||
pollUntilSettled,
|
||||
} from '../../../ai-video/providers/polling.js';
|
||||
import {
|
||||
closestAspectRatio,
|
||||
formatAspectRatio,
|
||||
@@ -145,6 +148,7 @@ export class KlingImageProvider implements IImageProvider {
|
||||
);
|
||||
if (!hold) throw insufficientCreditsError();
|
||||
|
||||
const jobHold = new JobHold(hold);
|
||||
try {
|
||||
const aspectRatio = params.imageSize
|
||||
? formatAspectRatio(
|
||||
@@ -184,7 +188,35 @@ export class KlingImageProvider implements IImageProvider {
|
||||
path,
|
||||
{ body, baseUrl: this.#baseUrl },
|
||||
);
|
||||
const task = await this.#pollUntilSettled(path, created);
|
||||
// Billed per image; a failed task isn't.
|
||||
const task = await pollUntilSettled<KlingImageTask>({
|
||||
provider: 'kling',
|
||||
providerLabel: 'Kling',
|
||||
noun: 'image',
|
||||
intervalMs: this.#pollIntervalMs,
|
||||
windowMs: POLL_WINDOW_MS,
|
||||
initial: created,
|
||||
fetch: () =>
|
||||
klingRequest<KlingImageTask>(
|
||||
this.#apiKey,
|
||||
'GET',
|
||||
`${path}/${encodeURIComponent(created.task_id)}`,
|
||||
{ baseUrl: this.#baseUrl },
|
||||
),
|
||||
isPending: (task) =>
|
||||
task.task_status === 'submitted' ||
|
||||
task.task_status === 'processing',
|
||||
meter: async (task, settled) => {
|
||||
if (settled && task?.task_status !== 'succeed') return;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
1,
|
||||
Math.round(cents * 1_000_000),
|
||||
);
|
||||
},
|
||||
hold: jobHold,
|
||||
});
|
||||
if (task.task_status !== 'succeed') {
|
||||
throw this.#taskFailure(task.task_status_msg);
|
||||
}
|
||||
@@ -192,68 +224,12 @@ export class KlingImageProvider implements IImageProvider {
|
||||
if (!url) {
|
||||
throw new Error('Kling response did not include an image URL');
|
||||
}
|
||||
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
1,
|
||||
Math.round(cents * 1_000_000),
|
||||
);
|
||||
return url;
|
||||
} finally {
|
||||
await hold.release();
|
||||
await jobHold.release();
|
||||
}
|
||||
}
|
||||
|
||||
async #pollUntilSettled(
|
||||
path: string,
|
||||
created: KlingImageTask,
|
||||
): Promise<KlingImageTask> {
|
||||
const signal = Context.get('abortSignal') as AbortSignal | undefined;
|
||||
const deadline = Date.now() + POLL_WINDOW_MS;
|
||||
let task = created;
|
||||
while (
|
||||
task.task_status === 'submitted' ||
|
||||
task.task_status === 'processing'
|
||||
) {
|
||||
if (signal?.aborted) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Client disconnected before image generation finished',
|
||||
{
|
||||
legacyCode: 'client_aborted',
|
||||
fields: { provider: 'kling' },
|
||||
},
|
||||
);
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
throw new HttpError(
|
||||
504,
|
||||
'Timed out waiting for Kling image generation to complete',
|
||||
{
|
||||
legacyCode: 'upstream_timeout',
|
||||
fields: { provider: 'kling' },
|
||||
},
|
||||
);
|
||||
}
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, this.#pollIntervalMs),
|
||||
);
|
||||
try {
|
||||
task = await klingRequest<KlingImageTask>(
|
||||
this.#apiKey,
|
||||
'GET',
|
||||
`${path}/${encodeURIComponent(created.task_id)}`,
|
||||
{ baseUrl: this.#baseUrl },
|
||||
);
|
||||
} catch (err) {
|
||||
// A missed poll is not a failed task.
|
||||
if (!isTransientUpstreamError(err)) throw err;
|
||||
}
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
/**
|
||||
* A content-policy refusal is the caller's to act on; anything else
|
||||
* Kling's.
|
||||
|
||||
@@ -47,7 +47,13 @@ import { PuterServer } from '../../server.js';
|
||||
import type { MeteringService } from '../../services/metering/MeteringService.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { generateDefaultFsentries } from '../../util/userProvisioning.js';
|
||||
import type { VoiceChangerDriver } from './VoiceChangerDriver.js';
|
||||
import type { DriverStreamResult } from '../meta.js';
|
||||
import {
|
||||
OggOpusClock,
|
||||
outputBytesPerSecond,
|
||||
type VoiceChangerDriver,
|
||||
wavDataOffset,
|
||||
} from './VoiceChangerDriver.js';
|
||||
import {
|
||||
callerError,
|
||||
readStreamResult,
|
||||
@@ -330,11 +336,11 @@ describe('VoiceChangerDriver.convert success path', () => {
|
||||
|
||||
it('meters one usage line at the per-second rate from costs.ts', async () => {
|
||||
const { actor } = await makeUser();
|
||||
// 32 KB of mp3_44100_128 back = 2 seconds.
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(new ArrayBuffer(8), 'audio/mpeg'),
|
||||
okResponse(new ArrayBuffer(32_000), 'audio/mpeg'),
|
||||
);
|
||||
|
||||
// 32 KB audio at 16 kbit/s = 2 seconds rounded up.
|
||||
const buf = Buffer.alloc(32_000);
|
||||
await readStreamResult(
|
||||
await withActor(actor, () =>
|
||||
@@ -344,7 +350,7 @@ describe('VoiceChangerDriver.convert success path', () => {
|
||||
|
||||
const usageType = 'elevenlabs:eleven_multilingual_sts_v2:second';
|
||||
const perSecond = VOICE_CHANGER_COSTS[usageType];
|
||||
const expectedSeconds = Math.max(1, Math.ceil(32_000 / 16000));
|
||||
const expectedSeconds = 2;
|
||||
|
||||
const calls = incrementUsageSpy.mock.calls.filter(
|
||||
([, type]) => type === usageType,
|
||||
@@ -365,7 +371,7 @@ describe('VoiceChangerDriver.convert success path', () => {
|
||||
.map(([, , , cost]) => cost);
|
||||
const convert = async (charges: number) => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(new ArrayBuffer(8), 'audio/mpeg'),
|
||||
okResponse(new ArrayBuffer(32_000), 'audio/mpeg'),
|
||||
);
|
||||
await readStreamResult(
|
||||
await withActor(actor, () =>
|
||||
@@ -385,6 +391,230 @@ describe('VoiceChangerDriver.convert success path', () => {
|
||||
expect(held).toEqual([base, base! * 2]);
|
||||
});
|
||||
|
||||
it.each<[string, number]>([
|
||||
['mp3_44100_128', 16_000],
|
||||
['mp3_22050_32', 4_000],
|
||||
['opus_48000_64', 8_000],
|
||||
['pcm_44100', 88_200],
|
||||
['ulaw_8000', 8_000],
|
||||
['wav_44100', 88_200],
|
||||
['mp3_44100_999999', 16_000],
|
||||
['flac_44100', 16_000],
|
||||
])('sizes %s at %s bytes per second', (format, bytes) => {
|
||||
expect(outputBytesPerSecond(format)).toBe(bytes);
|
||||
});
|
||||
|
||||
/** A 16-bit mono WAV header, with an optional LIST chunk before data. */
|
||||
const wavHeader = (sampleRate: number, dataBytes: number, list = 0) => {
|
||||
const listChunk = list
|
||||
? Buffer.concat([
|
||||
Buffer.from('LIST'),
|
||||
Buffer.from(new Uint32Array([list]).buffer),
|
||||
Buffer.alloc(list),
|
||||
])
|
||||
: Buffer.alloc(0);
|
||||
const fmt = Buffer.alloc(24);
|
||||
fmt.write('fmt ', 0, 'ascii');
|
||||
fmt.writeUInt32LE(16, 4);
|
||||
fmt.writeUInt16LE(1, 8);
|
||||
fmt.writeUInt16LE(1, 10);
|
||||
fmt.writeUInt32LE(sampleRate, 12);
|
||||
fmt.writeUInt32LE(sampleRate * 2, 16);
|
||||
fmt.writeUInt16LE(2, 20);
|
||||
fmt.writeUInt16LE(16, 22);
|
||||
const data = Buffer.alloc(8);
|
||||
data.write('data', 0, 'ascii');
|
||||
data.writeUInt32LE(dataBytes, 4);
|
||||
const body = Buffer.concat([fmt, listChunk, data]);
|
||||
const riff = Buffer.alloc(12);
|
||||
riff.write('RIFF', 0, 'ascii');
|
||||
riff.writeUInt32LE(4 + body.length + dataBytes, 4);
|
||||
riff.write('WAVE', 8, 'ascii');
|
||||
return Buffer.concat([riff, body]);
|
||||
};
|
||||
|
||||
it("finds where a WAV stream's samples start", () => {
|
||||
expect(wavDataOffset(wavHeader(16_000, 0))).toBe(44);
|
||||
expect(wavDataOffset(wavHeader(16_000, 0, 26))).toBe(78);
|
||||
expect(wavDataOffset(wavHeader(16_000, 0).subarray(0, 40))).toBe(
|
||||
undefined,
|
||||
);
|
||||
expect(wavDataOffset(Buffer.alloc(64))).toBe(undefined);
|
||||
});
|
||||
|
||||
/** One Ogg page carrying `body` as a single packet. */
|
||||
const oggPage = (granule: bigint, sequence: number, body: Buffer) => {
|
||||
const lacing: number[] = [];
|
||||
let left = body.length;
|
||||
while (left >= 255) {
|
||||
lacing.push(255);
|
||||
left -= 255;
|
||||
}
|
||||
lacing.push(left);
|
||||
const header = Buffer.alloc(27);
|
||||
header.write('OggS', 0, 'ascii');
|
||||
header.writeBigUInt64LE(granule, 6);
|
||||
header.writeUInt32LE(sequence, 18);
|
||||
header[26] = lacing.length;
|
||||
return Buffer.concat([header, Buffer.from(lacing), body]);
|
||||
};
|
||||
|
||||
/** A sparse Ogg Opus stream: `seconds` of audio in very few bytes. */
|
||||
const oggOpus = (seconds: number, preSkip = 312) => {
|
||||
const opusHead = Buffer.alloc(19);
|
||||
opusHead.write('OpusHead', 0, 'ascii');
|
||||
opusHead[8] = 1;
|
||||
opusHead[9] = 1;
|
||||
opusHead.writeUInt16LE(preSkip, 10);
|
||||
opusHead.writeUInt32LE(48_000, 12);
|
||||
const pages = [
|
||||
oggPage(0n, 0, opusHead),
|
||||
oggPage(0n, 1, Buffer.from('OpusTags')),
|
||||
];
|
||||
for (let s = 1; s <= seconds; s++) {
|
||||
pages.push(
|
||||
oggPage(BigInt(preSkip + s * 48_000), s + 1, Buffer.alloc(300)),
|
||||
);
|
||||
}
|
||||
return Buffer.concat(pages);
|
||||
};
|
||||
|
||||
it('times an Ogg Opus stream by its last granule, however it is chunked', () => {
|
||||
const stream = oggOpus(7);
|
||||
const whole = new OggOpusClock();
|
||||
whole.push(stream);
|
||||
expect(whole.seconds).toBe(7);
|
||||
|
||||
const trickle = new OggOpusClock();
|
||||
for (const byte of stream) trickle.push(Buffer.from([byte]));
|
||||
expect(trickle.seconds).toBe(7);
|
||||
});
|
||||
|
||||
it('does not time a stream that is not Ogg Opus', () => {
|
||||
const clock = new OggOpusClock();
|
||||
clock.push(Buffer.alloc(4096, 1));
|
||||
expect(clock.seconds).toBe(undefined);
|
||||
});
|
||||
|
||||
describe('meters the length of the audio that comes back', () => {
|
||||
const usageType = 'elevenlabs:eleven_multilingual_sts_v2:second';
|
||||
const billed = async () => {
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
incrementUsageSpy.mock.calls.filter(
|
||||
([, type]) => type === usageType,
|
||||
),
|
||||
).toHaveLength(1),
|
||||
);
|
||||
const [, , count, cost] = incrementUsageSpy.mock.calls.find(
|
||||
([, type]) => type === usageType,
|
||||
)!;
|
||||
expect(cost).toBe(VOICE_CHANGER_COSTS[usageType]! * count);
|
||||
return count;
|
||||
};
|
||||
const convert = async (
|
||||
actor: Actor,
|
||||
inputBytes: number,
|
||||
args: Record<string, unknown> = {},
|
||||
) =>
|
||||
readStreamResult(
|
||||
await withActor(actor, () =>
|
||||
driver.convert({
|
||||
audio: dataUrl(Buffer.alloc(inputBytes), 'audio/ogg'),
|
||||
...args,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
it('bills 600 s for a compact 600 s clip, not its byte size', async () => {
|
||||
const { actor } = await makeUser();
|
||||
// 600 s of Opus at 8 kbps in; 600 s of mp3_44100_128 out.
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(new ArrayBuffer(600 * 16_000)),
|
||||
);
|
||||
await convert(actor, 441_078);
|
||||
expect(await billed()).toBe(600);
|
||||
});
|
||||
|
||||
it('sizes the output by the requested output_format', async () => {
|
||||
const { actor } = await makeUser();
|
||||
// pcm_16000 is 16-bit mono: 32,000 bytes per second.
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(new ArrayBuffer(3 * 32_000), 'audio/pcm'),
|
||||
);
|
||||
await convert(actor, 1_000_000, { output_format: 'pcm_16000' });
|
||||
expect(await billed()).toBe(3);
|
||||
});
|
||||
|
||||
it('does not bill a WAV header as audio', async () => {
|
||||
const { actor } = await makeUser();
|
||||
// 3 s of wav_16000 is 96,000 bytes of samples plus its header.
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(
|
||||
new Uint8Array(
|
||||
Buffer.concat([
|
||||
wavHeader(16_000, 3 * 32_000, 26),
|
||||
Buffer.alloc(3 * 32_000),
|
||||
]),
|
||||
).buffer,
|
||||
'audio/wav',
|
||||
),
|
||||
);
|
||||
await convert(actor, 1_000_000, { output_format: 'wav_16000' });
|
||||
expect(await billed()).toBe(3);
|
||||
});
|
||||
|
||||
it('bills sparse Opus output by its duration, not its bytes', async () => {
|
||||
const { actor } = await makeUser();
|
||||
// 20 s of mostly silent opus_48000_64 is far under 20 s of bytes.
|
||||
const body = oggOpus(20);
|
||||
expect(body.length).toBeLessThan(8_000 * 2);
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(new Uint8Array(body).buffer, 'audio/ogg'),
|
||||
);
|
||||
await convert(actor, 1_000, { output_format: 'opus_48000_64' });
|
||||
expect(await billed()).toBe(20);
|
||||
});
|
||||
|
||||
it('sizes an undocumented output_format as the default', async () => {
|
||||
const { actor } = await makeUser();
|
||||
// 3 s at the default mp3_44100_128.
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
okResponse(new ArrayBuffer(3 * 16_000)),
|
||||
);
|
||||
await convert(actor, 1_000_000, {
|
||||
output_format: 'mp3_44100_999999',
|
||||
});
|
||||
expect(await billed()).toBe(3);
|
||||
});
|
||||
|
||||
it('charges at least the input estimate when the caller hangs up', async () => {
|
||||
const { actor } = await makeUser();
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(16_000));
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { 'content-type': 'audio/mpeg' } },
|
||||
),
|
||||
);
|
||||
const result = (await withActor(actor, () =>
|
||||
driver.convert({
|
||||
audio: dataUrl(Buffer.alloc(80_000), 'audio/mpeg'),
|
||||
}),
|
||||
)) as DriverStreamResult;
|
||||
for await (const _chunk of result.stream) break;
|
||||
expect(await billed()).toBe(5);
|
||||
await vi.waitFor(async () =>
|
||||
expect(
|
||||
await server.stores.creditHold.outstanding(actor.user.uuid),
|
||||
).toBe(0),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('meters nothing when ElevenLabs breaks off before any audio', async () => {
|
||||
const { actor } = await makeUser();
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
|
||||
@@ -17,9 +17,11 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { pipeline, Readable, Transform } from 'node:stream';
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import type { DriverStreamResult } from '../meta.js';
|
||||
import { OUTPUT_FORMATS } from '../ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.js';
|
||||
import { PuterDriver } from '../types.js';
|
||||
import {
|
||||
type AiMeteringService,
|
||||
@@ -198,10 +200,9 @@ export class VoiceChangerDriver extends PuterDriver {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
|
||||
// Metering: estimate duration from file size if we don't parse metadata.
|
||||
// 16 kbit/s is a safe lower bound for speech audio; pre-check credits
|
||||
// before we hit the ElevenLabs API. Post-usage we increment by the same
|
||||
// estimate — duration parsing is deferred to v2.1 if needed.
|
||||
// The hold estimates seconds from the input at 128 kbps: compact audio
|
||||
// is under-held rather than refused. Usage is metered from the length
|
||||
// of the audio that comes back.
|
||||
const estimatedSeconds = Math.max(
|
||||
1,
|
||||
Math.ceil(loaded.buffer.byteLength / 16000),
|
||||
@@ -263,6 +264,10 @@ export class VoiceChangerDriver extends PuterDriver {
|
||||
if (search) url.search = search;
|
||||
|
||||
const metering = this.#aiMetering;
|
||||
const output = measuredOutput(
|
||||
outputBytesPerSecond(outputFormat),
|
||||
outputFormat,
|
||||
);
|
||||
return withStreamedAiCharge(
|
||||
metering,
|
||||
actor,
|
||||
@@ -287,16 +292,206 @@ export class VoiceChangerDriver extends PuterDriver {
|
||||
dataType: 'stream',
|
||||
content_type:
|
||||
response.headers.get('content-type') ?? 'audio/mpeg',
|
||||
stream: upstreamBodyStream(response),
|
||||
stream: output.measure(upstreamBodyStream(response)),
|
||||
};
|
||||
},
|
||||
() =>
|
||||
metering.incrementUsage(
|
||||
async () => {
|
||||
const measured = await output.seconds;
|
||||
// A stream cut short was still converted in full upstream.
|
||||
const seconds =
|
||||
measured.complete && measured.seconds
|
||||
? measured.seconds
|
||||
: Math.max(measured.seconds ?? 0, estimatedSeconds);
|
||||
return metering.incrementUsage(
|
||||
actor,
|
||||
usageKey,
|
||||
estimatedSeconds,
|
||||
estimatedCost,
|
||||
),
|
||||
seconds,
|
||||
ucentsPerSecond * seconds,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes per second of an ElevenLabs `output_format`. A format outside the
|
||||
* documented set is sized as the default, so a made-up bitrate can't shrink the
|
||||
* bill.
|
||||
*/
|
||||
export function outputBytesPerSecond(format: string): number {
|
||||
const known = OUTPUT_FORMATS.has(format) ? format : DEFAULT_OUTPUT_FORMAT;
|
||||
const [codec, rate, kbps] = known.split('_');
|
||||
if (codec === 'mp3' || codec === 'opus') return Number(kbps) * 125;
|
||||
if (codec === 'ulaw' || codec === 'alaw') return Number(rate);
|
||||
// pcm and wav: 16-bit mono.
|
||||
return Number(rate) * 2;
|
||||
}
|
||||
|
||||
// Past this, a WAV whose data chunk hasn't started is billed by raw length.
|
||||
const WAV_HEADER_MAX_BYTES = 64 * 1024;
|
||||
|
||||
/**
|
||||
* Where the samples of a RIFF/WAVE stream start, once `head` reaches its `data`
|
||||
* chunk; undefined until then, or if `head` isn't a WAV.
|
||||
*/
|
||||
export function wavDataOffset(head: Buffer): number | undefined {
|
||||
if (head.length < 12) return undefined;
|
||||
if (
|
||||
head.toString('ascii', 0, 4) !== 'RIFF' ||
|
||||
head.toString('ascii', 8, 12) !== 'WAVE'
|
||||
)
|
||||
return undefined;
|
||||
let offset = 12;
|
||||
while (offset + 8 <= head.length) {
|
||||
if (head.toString('ascii', offset, offset + 4) === 'data')
|
||||
return offset + 8;
|
||||
const size = head.readUInt32LE(offset + 4);
|
||||
offset += 8 + size + (size & 1);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const OGG_HEADER_BYTES = 27;
|
||||
const NO_GRANULE = 2n ** 64n - 1n;
|
||||
|
||||
/**
|
||||
* Walks an Ogg Opus stream page by page as it arrives. Opus is variable
|
||||
* bitrate, so its length comes from the last page's granule position (48 kHz
|
||||
* samples, less the stream's pre-skip) rather than its byte count.
|
||||
*/
|
||||
export class OggOpusClock {
|
||||
#header = Buffer.alloc(0);
|
||||
#skip = 0;
|
||||
#firstBody: Buffer | undefined = Buffer.alloc(0);
|
||||
#firstBodyBytes = 0;
|
||||
#granule: bigint | undefined;
|
||||
#preSkip = 0;
|
||||
#broken = false;
|
||||
|
||||
push(chunk: Buffer): void {
|
||||
let offset = 0;
|
||||
while (!this.#broken && offset < chunk.length) {
|
||||
if (this.#skip > 0) {
|
||||
const n = Math.min(this.#skip, chunk.length - offset);
|
||||
if (this.#firstBody && this.#firstBody.length < 19) {
|
||||
this.#firstBody = Buffer.concat([
|
||||
this.#firstBody,
|
||||
chunk.subarray(offset, offset + n),
|
||||
]);
|
||||
}
|
||||
this.#skip -= n;
|
||||
offset += n;
|
||||
if (this.#skip === 0 && this.#firstBody) this.#readOpusHead();
|
||||
continue;
|
||||
}
|
||||
const want =
|
||||
this.#header.length < OGG_HEADER_BYTES
|
||||
? OGG_HEADER_BYTES
|
||||
: OGG_HEADER_BYTES + this.#header[26]!;
|
||||
const n = Math.min(
|
||||
want - this.#header.length,
|
||||
chunk.length - offset,
|
||||
);
|
||||
this.#header = Buffer.concat([
|
||||
this.#header,
|
||||
chunk.subarray(offset, offset + n),
|
||||
]);
|
||||
offset += n;
|
||||
if (this.#header.length < want) continue;
|
||||
if (this.#header.toString('ascii', 0, 4) !== 'OggS') {
|
||||
this.#broken = true;
|
||||
return;
|
||||
}
|
||||
if (want === OGG_HEADER_BYTES && this.#header[26]! > 0) continue;
|
||||
const granule = this.#header.readBigUInt64LE(6);
|
||||
if (granule !== NO_GRANULE) this.#granule = granule;
|
||||
let body = 0;
|
||||
for (const lacing of this.#header.subarray(OGG_HEADER_BYTES)) {
|
||||
body += lacing;
|
||||
}
|
||||
this.#header = Buffer.alloc(0);
|
||||
this.#skip = body;
|
||||
if (this.#firstBody) this.#firstBodyBytes = body;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whole seconds so far, once the stream has proven to be Ogg Opus. */
|
||||
get seconds(): number | undefined {
|
||||
if (this.#broken || this.#granule === undefined) return undefined;
|
||||
const samples = Number(this.#granule) - this.#preSkip;
|
||||
return samples > 0
|
||||
? Math.max(1, Math.ceil(samples / 48_000))
|
||||
: undefined;
|
||||
}
|
||||
|
||||
#readOpusHead(): void {
|
||||
const head = this.#firstBody!;
|
||||
this.#firstBody = undefined;
|
||||
if (
|
||||
this.#firstBodyBytes < 19 ||
|
||||
head.toString('ascii', 0, 8) !== 'OpusHead'
|
||||
) {
|
||||
this.#broken = true;
|
||||
return;
|
||||
}
|
||||
this.#preSkip = head.readUInt16LE(10);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the bytes of a converted stream. `seconds` settles when the stream
|
||||
* ends or closes: its length in whole seconds (undefined when nothing came
|
||||
* back), and whether the upstream sent all of it. A WAV's header isn't audio,
|
||||
* so it isn't counted; Opus is timed by its pages.
|
||||
*/
|
||||
function measuredOutput(bytesPerSecond: number, format: string) {
|
||||
let bytes = 0;
|
||||
let head: Buffer | undefined = format.startsWith('wav_')
|
||||
? Buffer.alloc(0)
|
||||
: undefined;
|
||||
const opus = format.startsWith('opus_') ? new OggOpusClock() : undefined;
|
||||
let headerBytes = 0;
|
||||
let settle!: (measured: { seconds?: number; complete: boolean }) => void;
|
||||
const seconds = new Promise<{ seconds?: number; complete: boolean }>(
|
||||
(resolve) => {
|
||||
settle = resolve;
|
||||
},
|
||||
);
|
||||
const lengthOf = () => {
|
||||
const timed = opus?.seconds;
|
||||
if (timed !== undefined) return timed;
|
||||
const audio = bytes - headerBytes;
|
||||
return audio > 0
|
||||
? Math.max(1, Math.ceil(audio / bytesPerSecond))
|
||||
: undefined;
|
||||
};
|
||||
const measure = (source: Readable): Readable => {
|
||||
const counter = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
bytes += chunk.length;
|
||||
opus?.push(chunk);
|
||||
if (head) {
|
||||
head = Buffer.concat([head, chunk]);
|
||||
const offset = wavDataOffset(head);
|
||||
if (offset !== undefined) headerBytes = offset;
|
||||
if (
|
||||
offset !== undefined ||
|
||||
head.length >= WAV_HEADER_MAX_BYTES
|
||||
)
|
||||
head = undefined;
|
||||
}
|
||||
callback(null, chunk);
|
||||
},
|
||||
flush(callback) {
|
||||
settle({ seconds: lengthOf(), complete: true });
|
||||
callback();
|
||||
},
|
||||
});
|
||||
counter.once('close', () =>
|
||||
settle({ seconds: lengthOf(), complete: false }),
|
||||
);
|
||||
pipeline(source, counter, () => {});
|
||||
return counter;
|
||||
};
|
||||
return { measure, seconds };
|
||||
}
|
||||
@@ -508,6 +508,26 @@ describe('SpeechToTextDriver model selection and payload shape', () => {
|
||||
expect(sent.temperature).toBe(0.2);
|
||||
});
|
||||
|
||||
it('keeps extra_body from overriding the response shape billing reads', async () => {
|
||||
const { actor } = await makeUser();
|
||||
transcriptionsCreateMock.mockResolvedValueOnce({ text: 'x' });
|
||||
|
||||
await withActor(actor, () =>
|
||||
driver.transcribe({
|
||||
file: dataUrl(Buffer.from('a'), 'audio/mp3'),
|
||||
model: 'gpt-transcribe',
|
||||
extra_body: {
|
||||
response_format: 'text',
|
||||
stream: true,
|
||||
chunking_strategy: 'auto',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const sent = transcriptionsCreateMock.mock.calls[0]![0];
|
||||
expect(sent.extra_body).toEqual({ chunking_strategy: 'auto' });
|
||||
});
|
||||
|
||||
it('does not forward timestamp_granularities, which gpt-transcribe ignores', async () => {
|
||||
const { actor } = await makeUser();
|
||||
transcriptionsCreateMock.mockResolvedValueOnce({ text: 'x' });
|
||||
@@ -684,6 +704,72 @@ describe('SpeechToTextDriver metering', () => {
|
||||
expect(creditCall[1]).toBe(expected);
|
||||
});
|
||||
|
||||
it('meters the duration OpenAI reports, not the byte estimate', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const usageType = 'openai:gpt-transcribe:second';
|
||||
const billed = async (bytes: number, seconds: number) => {
|
||||
incrementUsageSpy.mockClear();
|
||||
transcriptionsCreateMock.mockResolvedValueOnce({
|
||||
text: 'x',
|
||||
usage: { type: 'duration', seconds },
|
||||
});
|
||||
await withActor(actor, () =>
|
||||
driver.transcribe({
|
||||
file: dataUrl(Buffer.alloc(bytes, 1), 'audio/ogg'),
|
||||
}),
|
||||
);
|
||||
const [, , count, cost] = incrementUsageSpy.mock.calls.find(
|
||||
([, type]) => type === usageType,
|
||||
)!;
|
||||
expect(cost).toBe(SPEECH_TO_TEXT_COSTS[usageType]! * count);
|
||||
return count;
|
||||
};
|
||||
// 600 s of Opus at 8 kbps, and 60 s of 44.1 kHz stereo WAV.
|
||||
expect(await billed(441_078, 600)).toBe(600);
|
||||
expect(await billed(10_584_078, 59.2)).toBe(60);
|
||||
});
|
||||
|
||||
it('asks for json behind response_format=text so usage comes back', async () => {
|
||||
const { actor } = await makeUser();
|
||||
transcriptionsCreateMock.mockResolvedValueOnce({
|
||||
text: 'plain transcript',
|
||||
usage: { type: 'duration', seconds: 42 },
|
||||
});
|
||||
const result = await withActor(actor, () =>
|
||||
driver.transcribe({
|
||||
file: dataUrl(Buffer.from('a'), 'audio/mp3'),
|
||||
response_format: 'text',
|
||||
}),
|
||||
);
|
||||
expect(result).toBe('plain transcript');
|
||||
expect(transcriptionsCreateMock.mock.calls[0]![0].response_format).toBe(
|
||||
'json',
|
||||
);
|
||||
const usageType = 'openai:gpt-transcribe:second';
|
||||
const [, , count] = incrementUsageSpy.mock.calls.find(
|
||||
([, type]) => type === usageType,
|
||||
)!;
|
||||
expect(count).toBe(42);
|
||||
});
|
||||
|
||||
it('falls back to the byte estimate when OpenAI reports token usage', async () => {
|
||||
const { actor } = await makeUser();
|
||||
transcriptionsCreateMock.mockResolvedValueOnce({
|
||||
text: 'x',
|
||||
usage: { type: 'tokens', input_tokens: 10, output_tokens: 2 },
|
||||
});
|
||||
await withActor(actor, () =>
|
||||
driver.transcribe({
|
||||
file: dataUrl(Buffer.alloc(32000, 0), 'audio/mp3'),
|
||||
}),
|
||||
);
|
||||
const usageType = 'openai:gpt-transcribe:second';
|
||||
const [, , count] = incrementUsageSpy.mock.calls.find(
|
||||
([, type]) => type === usageType,
|
||||
)!;
|
||||
expect(count).toBe(2);
|
||||
});
|
||||
|
||||
it('holds and meters at the AI cost factor', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const usageType = 'openai:gpt-transcribe:second';
|
||||
|
||||
@@ -170,9 +170,9 @@ export class OpenAISpeechToTextProvider extends SpeechToTextProvider {
|
||||
);
|
||||
}
|
||||
|
||||
// Estimate seconds from raw bytes — 16 kbps is a conservative speech-audio
|
||||
// lower bound. Full metadata parsing (music-metadata) is deferred — clients
|
||||
// aren't observably sensitive to billing-time delta vs real duration.
|
||||
// The hold estimates seconds from bytes at 128 kbps: compact audio is
|
||||
// under-held rather than refused, and usage is metered from the
|
||||
// duration OpenAI reports.
|
||||
const estimatedSeconds = Math.max(
|
||||
1,
|
||||
Math.ceil(loaded.buffer.byteLength / 16000),
|
||||
@@ -198,8 +198,13 @@ export class OpenAISpeechToTextProvider extends SpeechToTextProvider {
|
||||
file: openaiFile,
|
||||
model: selectedModel,
|
||||
};
|
||||
// A `text` response carries no usage, so json is asked for and
|
||||
// its text returned.
|
||||
if (args.response_format)
|
||||
payload.response_format = args.response_format;
|
||||
payload.response_format =
|
||||
args.response_format === 'text'
|
||||
? 'json'
|
||||
: args.response_format;
|
||||
if (args.language) payload.language = args.language;
|
||||
if (typeof args.temperature === 'number')
|
||||
payload.temperature = args.temperature;
|
||||
@@ -207,7 +212,15 @@ export class OpenAISpeechToTextProvider extends SpeechToTextProvider {
|
||||
// The API ignores a bare `logprobs` flag; it only honors `include`.
|
||||
if (args.logprobs && caps.canLogprobs)
|
||||
payload.include = ['logprobs'];
|
||||
if (args.extra_body) payload.extra_body = args.extra_body;
|
||||
if (args.extra_body && typeof args.extra_body === 'object') {
|
||||
// Billing reads usage off the response; these can strip it.
|
||||
const extra = {
|
||||
...(args.extra_body as Record<string, unknown>),
|
||||
};
|
||||
delete extra.response_format;
|
||||
delete extra.stream;
|
||||
payload.extra_body = extra;
|
||||
}
|
||||
|
||||
const result = await this.#openai.audio.transcriptions.create(
|
||||
payload as unknown as Parameters<
|
||||
@@ -215,11 +228,12 @@ export class OpenAISpeechToTextProvider extends SpeechToTextProvider {
|
||||
>[0],
|
||||
);
|
||||
|
||||
const billedSeconds = reportedSeconds(result) ?? estimatedSeconds;
|
||||
this.deps.metering.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
estimatedSeconds,
|
||||
ucentsPerSecond * estimatedSeconds,
|
||||
billedSeconds,
|
||||
ucentsPerSecond * billedSeconds,
|
||||
);
|
||||
|
||||
// Text response_format: return raw string; otherwise forward the OpenAI object.
|
||||
@@ -234,3 +248,18 @@ export class OpenAISpeechToTextProvider extends SpeechToTextProvider {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Whole seconds of audio OpenAI reports billing for, if it reports them. */
|
||||
function reportedSeconds(result: unknown): number | undefined {
|
||||
if (!result || typeof result !== 'object') return undefined;
|
||||
const { usage, duration } = result as {
|
||||
usage?: { type?: string; seconds?: unknown };
|
||||
duration?: unknown;
|
||||
};
|
||||
const seconds = usage?.type === 'duration' ? usage.seconds : duration;
|
||||
return typeof seconds === 'number' &&
|
||||
Number.isFinite(seconds) &&
|
||||
seconds > 0
|
||||
? Math.max(1, Math.ceil(seconds))
|
||||
: undefined;
|
||||
}
|
||||
@@ -48,7 +48,7 @@ const VOICE_ID_PATTERN = /^[A-Za-z0-9]{20}$/;
|
||||
const VOICE_SEARCH_MAX_PAGES = 5;
|
||||
// https://elevenlabs.io/docs/api-reference/text-to-speech/convert
|
||||
// mp3_44100_192 needs the Creator tier; pcm/wav at 44.1 kHz need Pro.
|
||||
const OUTPUT_FORMATS = new Set([
|
||||
export const OUTPUT_FORMATS = new Set([
|
||||
'alaw_8000',
|
||||
'mp3_22050_32',
|
||||
'mp3_24000_48',
|
||||
|
||||
@@ -41,7 +41,8 @@ import {
|
||||
} from 'vitest';
|
||||
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { Context, runWithContext } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { type Actor, SYSTEM_ACTOR } from '../../core/actor.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import type { MeteringService } from '../../services/metering/MeteringService.js';
|
||||
@@ -926,6 +927,29 @@ describe('VideoGenerationDriver.generate puter_output_path', () => {
|
||||
expect(result).toBe(`data:video/mp4;base64,${videoBytes}`);
|
||||
});
|
||||
|
||||
it('returns the result and flags it unsaved when write access is revoked mid-job', async () => {
|
||||
vi.spyOn(server.services.acl, 'check').mockResolvedValueOnce(true);
|
||||
vi.spyOn(server.services.fs, 'checkFSAccess').mockRejectedValueOnce(
|
||||
new HttpError(403, 'Access denied', { legacyCode: 'forbidden' }),
|
||||
);
|
||||
const fsWriteSpy = vi.spyOn(server.services.fs, 'write');
|
||||
const videoBytes = Buffer.from('video-bytes').toString('base64');
|
||||
completeTogetherJob(`data:video/mp4;base64,${videoBytes}`);
|
||||
|
||||
const [result, metadata] = await withTestUser(async () => [
|
||||
await driver.generate({
|
||||
prompt: 'hi',
|
||||
model: DEFAULT_MODEL,
|
||||
puter_output_path: '/testuser/videos/clip.mp4',
|
||||
} as never),
|
||||
Context.get('driverMetadata'),
|
||||
]);
|
||||
|
||||
expect(result).toBe(`data:video/mp4;base64,${videoBytes}`);
|
||||
expect(fsWriteSpy).not.toHaveBeenCalled();
|
||||
expect(metadata).toEqual({ outputSaved: false });
|
||||
});
|
||||
|
||||
it('does not forward puter_output_path to the upstream provider call', async () => {
|
||||
const aclCheckSpy = vi.spyOn(server.services.acl, 'check');
|
||||
aclCheckSpy.mockResolvedValueOnce(true);
|
||||
|
||||
@@ -336,13 +336,16 @@ export class VideoGenerationDriver extends PuterDriver {
|
||||
});
|
||||
|
||||
if (resolvedOutputPath) {
|
||||
await saveGeneratedMediaToFS(
|
||||
// False when write access was revoked while the job ran; the
|
||||
// result still comes back.
|
||||
const outputSaved = await saveGeneratedMediaToFS(
|
||||
this.services.fs,
|
||||
actor,
|
||||
result,
|
||||
resolvedOutputPath,
|
||||
{ noun: 'video', defaultType: 'video/mp4' },
|
||||
);
|
||||
Context.set('driverMetadata', { outputSaved });
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -395,25 +395,33 @@ describe('BytePlusVideoProvider.generate polling and outcomes', () => {
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(4); // 1 create + 3 polls
|
||||
});
|
||||
|
||||
it('gives up after the wait window as HttpError 504 upstream_timeout, without metering', async () => {
|
||||
it('gives up after the wait window as HttpError 504 upstream_timeout, metering the task once it settles', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let late: Record<string, unknown> = {
|
||||
id: 'cgt-slow',
|
||||
status: 'running',
|
||||
};
|
||||
fetchSpy
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({ id: 'cgt-slow', status: 'queued' }),
|
||||
)
|
||||
.mockImplementation(async () =>
|
||||
jsonResponse({ id: 'cgt-slow', status: 'running' }),
|
||||
);
|
||||
.mockImplementation(async () => jsonResponse(late));
|
||||
// The default poll interval keeps the window to ~60 polls.
|
||||
const provider = new BytePlusVideoProvider(
|
||||
{ apiKey: 'test-key' },
|
||||
aiMetering(),
|
||||
);
|
||||
|
||||
const rejection = withTestActor(() =>
|
||||
provider.generate({ prompt: 'hi' }),
|
||||
).catch((e: unknown) => e);
|
||||
const rejection = (async () => {
|
||||
try {
|
||||
await withTestActor(() =>
|
||||
provider.generate({ prompt: 'hi' }),
|
||||
);
|
||||
} catch (e) {
|
||||
return e;
|
||||
}
|
||||
})();
|
||||
|
||||
// Ten-minute wait window, polled every 5s.
|
||||
await vi.advanceTimersByTimeAsync(VIDEO_POLL_WINDOW_MS + 5_000);
|
||||
@@ -425,7 +433,21 @@ describe('BytePlusVideoProvider.generate polling and outcomes', () => {
|
||||
'Timed out waiting for BytePlus video generation to complete',
|
||||
fields: { provider: 'byteplus' },
|
||||
});
|
||||
// BytePlus still renders and bills the task past the window.
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
late = {
|
||||
id: 'cgt-slow',
|
||||
status: 'succeeded',
|
||||
content: { video_url: 'https://byteplus.example/v.mp4' },
|
||||
usage: { completion_tokens: 1234 },
|
||||
};
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
const [, usageType, tokens, cost] =
|
||||
incrementUsageSpy.mock.calls[0]!;
|
||||
expect(usageType).toMatch(/^byteplus-video-generation:/);
|
||||
expect(tokens).toBe(1234);
|
||||
expect(cost).toBeGreaterThan(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import type { IGenerateVideoParams, IVideoModel } from '../../types.js';
|
||||
import { capSecondsToRemainingCredits } from '../../creditCap.js';
|
||||
import { VideoProvider } from '../VideoProvider.js';
|
||||
import { pollUntilSettled, videoJobFailure } from '../polling.js';
|
||||
import { JobHold, pollUntilSettled, videoJobFailure } from '../polling.js';
|
||||
import {
|
||||
BYTEPLUS_VIDEO_GENERATION_MODELS,
|
||||
BYTEPLUS_VIDEO_SPECS,
|
||||
@@ -171,6 +171,7 @@ export class BytePlusVideoProvider extends VideoProvider {
|
||||
allowedSeconds: model.durationSeconds,
|
||||
});
|
||||
|
||||
const jobHold = new JobHold(hold);
|
||||
try {
|
||||
const body: Record<string, unknown> = {
|
||||
model: model.id,
|
||||
@@ -198,8 +199,44 @@ export class BytePlusVideoProvider extends VideoProvider {
|
||||
body.seed = Math.round(seed);
|
||||
}
|
||||
|
||||
// Bills the tokens the task reports. A finished task without
|
||||
// usage falls back to the estimate, a failed one to nothing.
|
||||
const meter = async (
|
||||
finalTask: ArkVideoTask | undefined,
|
||||
settled: boolean,
|
||||
) => {
|
||||
const finalCostKey = this.#costKey(
|
||||
model,
|
||||
this.#normalizeResolution(
|
||||
finalTask?.resolution,
|
||||
model,
|
||||
resolutionKey,
|
||||
),
|
||||
audioOn,
|
||||
);
|
||||
const finalCentsPerToken =
|
||||
model.costs?.[finalCostKey] ?? centsPerToken;
|
||||
const estimate = Math.round(tokensPerSecond * cappedSeconds);
|
||||
const tokens =
|
||||
finalTask?.usage?.completion_tokens ??
|
||||
(finalTask?.status === 'succeeded' || !settled
|
||||
? estimate
|
||||
: 0);
|
||||
if (!tokens) return;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
`byteplus-video-generation:${model.id}:${finalCostKey}`,
|
||||
tokens,
|
||||
tokens * finalCentsPerToken * 1_000_000,
|
||||
);
|
||||
};
|
||||
|
||||
const task = await this.#createTask(body);
|
||||
const finalTask = await this.#pollUntilComplete(task.id);
|
||||
const finalTask = await this.#pollUntilComplete(
|
||||
task.id,
|
||||
meter,
|
||||
jobHold,
|
||||
);
|
||||
|
||||
if (finalTask.status !== 'succeeded') {
|
||||
const errorMessage =
|
||||
@@ -219,33 +256,9 @@ export class BytePlusVideoProvider extends VideoProvider {
|
||||
);
|
||||
}
|
||||
|
||||
// Bill the tokens the task actually reports; fall back to the
|
||||
// pre-flight estimate if usage is missing.
|
||||
const finalResolutionKey = this.#normalizeResolution(
|
||||
finalTask.resolution,
|
||||
model,
|
||||
resolutionKey,
|
||||
);
|
||||
const finalCostKey = this.#costKey(
|
||||
model,
|
||||
finalResolutionKey,
|
||||
audioOn,
|
||||
);
|
||||
const finalCentsPerToken =
|
||||
model.costs?.[finalCostKey] ?? centsPerToken;
|
||||
const tokens =
|
||||
finalTask.usage?.completion_tokens ??
|
||||
Math.round(tokensPerSecond * cappedSeconds);
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
`byteplus-video-generation:${model.id}:${finalCostKey}`,
|
||||
tokens,
|
||||
tokens * finalCentsPerToken * 1_000_000,
|
||||
);
|
||||
|
||||
return videoUrl;
|
||||
} finally {
|
||||
await hold.release();
|
||||
await jobHold.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -345,7 +358,14 @@ export class BytePlusVideoProvider extends VideoProvider {
|
||||
})) as ArkVideoTask;
|
||||
}
|
||||
|
||||
async #pollUntilComplete(taskId: string): Promise<ArkVideoTask> {
|
||||
async #pollUntilComplete(
|
||||
taskId: string,
|
||||
meter: (
|
||||
task: ArkVideoTask | undefined,
|
||||
settled: boolean,
|
||||
) => Promise<unknown>,
|
||||
hold: JobHold,
|
||||
): Promise<ArkVideoTask> {
|
||||
return await pollUntilSettled<ArkVideoTask>({
|
||||
provider: 'byteplus',
|
||||
providerLabel: 'BytePlus',
|
||||
@@ -357,6 +377,8 @@ export class BytePlusVideoProvider extends VideoProvider {
|
||||
) as Promise<ArkVideoTask>,
|
||||
isPending: (task) =>
|
||||
task.status === 'queued' || task.status === 'running',
|
||||
meter,
|
||||
hold,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -37,12 +37,14 @@ import {
|
||||
type MockInstance,
|
||||
} from 'vitest';
|
||||
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../../integrationTestUtil.js';
|
||||
import { withAiCostFactor } from '../../../util/aiCostFactor.js';
|
||||
import type { IGenerateVideoParams } from '../../types.js';
|
||||
import { VIDEO_POLL_WINDOW_MS } from '../polling.js';
|
||||
import { KlingVideoProvider } from './KlingVideoProvider.js';
|
||||
import { KLING_VIDEO_GENERATION_MODELS } from './models.js';
|
||||
|
||||
@@ -663,9 +665,82 @@ describe('KlingVideoProvider failures', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('fails when the finished task has no video', async () => {
|
||||
// Kling bills a task it reports as succeeded, video or not.
|
||||
it('fails, still metered, when the finished task has no video', async () => {
|
||||
mockTaskFlow(succeededTask({ outputs: [] }));
|
||||
await expect(generate({})).rejects.toThrow(/video URL/);
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('KlingVideoProvider after the caller leaves', () => {
|
||||
it('keeps polling after a disconnect and meters the finished clip', async () => {
|
||||
mockTaskFlow({ id: 'task-1', status: 'processing' }, succeededTask());
|
||||
const result = await withTestActor(() => {
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return makeProvider().generate({ prompt: 'a ball rolls' });
|
||||
});
|
||||
expect(result).toBe('https://kling.example/out.mp4');
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
/** Times out on the first status poll; later polls return `late()`. */
|
||||
const outliveWindow = async (late: () => unknown) => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
jsonResponse(envelope({ id: 'task-1', status: 'submitted' })),
|
||||
);
|
||||
const realNow = Date.now.bind(Date);
|
||||
let skew = 0;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => realNow() + skew);
|
||||
let polls = 0;
|
||||
fetchSpy.mockImplementation(async () => {
|
||||
skew = VIDEO_POLL_WINDOW_MS + 1;
|
||||
return jsonResponse(
|
||||
envelope([
|
||||
polls++ === 0
|
||||
? { id: 'task-1', status: 'processing' }
|
||||
: late(),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
let failure: unknown;
|
||||
try {
|
||||
await generate({ seconds: 5 });
|
||||
} catch (e) {
|
||||
failure = e;
|
||||
}
|
||||
expect(failure).toMatchObject({
|
||||
statusCode: 504,
|
||||
legacyCode: 'upstream_timeout',
|
||||
});
|
||||
};
|
||||
|
||||
it('meters a task that finishes after the window', async () => {
|
||||
let finished = false;
|
||||
await outliveWindow(() =>
|
||||
finished ? succeededTask() : { id: 'task-1', status: 'processing' },
|
||||
);
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
|
||||
finished = true;
|
||||
await vi.waitFor(() =>
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1),
|
||||
);
|
||||
const [, usageType, amount] = incrementUsageSpy.mock.calls[0]!;
|
||||
expect(usageType).toMatch(/^kling-video-generation:/);
|
||||
expect(amount).toBe(5);
|
||||
});
|
||||
|
||||
it('meters nothing for a task that fails after the window', async () => {
|
||||
await outliveWindow(() => ({
|
||||
id: 'task-1',
|
||||
status: 'failed',
|
||||
message: 'boom',
|
||||
}));
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,7 @@ import { capSecondsToRemainingCredits } from '../../creditCap.js';
|
||||
import type { IGenerateVideoParams, IVideoModel } from '../../types.js';
|
||||
import { VideoProvider } from '../VideoProvider.js';
|
||||
import {
|
||||
JobHold,
|
||||
pollUntilSettled,
|
||||
VIDEO_HOLD_TTL_MS,
|
||||
videoJobFailure,
|
||||
@@ -240,6 +241,7 @@ export class KlingVideoProvider extends VideoProvider {
|
||||
allowedSeconds,
|
||||
});
|
||||
|
||||
const jobHold = new JobHold(hold);
|
||||
try {
|
||||
const sourceVideo = hasVideo
|
||||
? await this.#sourceVideo(actor, inputVideo as string)
|
||||
@@ -262,7 +264,36 @@ export class KlingVideoProvider extends VideoProvider {
|
||||
path,
|
||||
{ body, baseUrl: this.#baseUrl },
|
||||
);
|
||||
const task = await this.#pollUntilComplete(created.id);
|
||||
// Kling bills whole seconds of the clip it returns: an edit's
|
||||
// length comes from its source, everything else from the
|
||||
// request. A failed task isn't billed.
|
||||
const meter = async (
|
||||
task: KlingTask | undefined,
|
||||
settled: boolean,
|
||||
) => {
|
||||
if (settled && task?.status !== 'succeeded') return;
|
||||
const video = task?.outputs?.find(
|
||||
(output) => output.type === 'video' && output.url,
|
||||
);
|
||||
const outputSeconds = Number.parseFloat(video?.duration ?? '');
|
||||
const billedSeconds =
|
||||
hasVideo &&
|
||||
Number.isFinite(outputSeconds) &&
|
||||
outputSeconds > 0
|
||||
? Math.max(1, Math.round(outputSeconds))
|
||||
: cappedSeconds;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
billedSeconds,
|
||||
Math.round(billedSeconds * centsPerSecond * 1_000_000),
|
||||
);
|
||||
};
|
||||
const task = await this.#pollUntilComplete(
|
||||
created.id,
|
||||
meter,
|
||||
jobHold,
|
||||
);
|
||||
if (
|
||||
task.status !== 'succeeded' &&
|
||||
sourceVideo &&
|
||||
@@ -283,24 +314,9 @@ export class KlingVideoProvider extends VideoProvider {
|
||||
if (!video?.url) {
|
||||
throw new Error('Kling response did not include a video URL');
|
||||
}
|
||||
|
||||
// Kling bills whole seconds of the clip it returns: an edit's
|
||||
// length comes from its source, everything else from the request.
|
||||
const outputSeconds = Number.parseFloat(video.duration ?? '');
|
||||
const billedSeconds =
|
||||
hasVideo && Number.isFinite(outputSeconds) && outputSeconds > 0
|
||||
? Math.max(1, Math.round(outputSeconds))
|
||||
: cappedSeconds;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
billedSeconds,
|
||||
Math.round(billedSeconds * centsPerSecond * 1_000_000),
|
||||
);
|
||||
|
||||
return video.url;
|
||||
} finally {
|
||||
await hold.release();
|
||||
await jobHold.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -476,7 +492,14 @@ export class KlingVideoProvider extends VideoProvider {
|
||||
});
|
||||
}
|
||||
|
||||
async #pollUntilComplete(taskId: string): Promise<KlingTask> {
|
||||
async #pollUntilComplete(
|
||||
taskId: string,
|
||||
meter: (
|
||||
task: KlingTask | undefined,
|
||||
settled: boolean,
|
||||
) => Promise<unknown>,
|
||||
hold: JobHold,
|
||||
): Promise<KlingTask> {
|
||||
return pollUntilSettled<KlingTask>({
|
||||
provider: 'kling',
|
||||
providerLabel: 'Kling',
|
||||
@@ -496,6 +519,8 @@ export class KlingVideoProvider extends VideoProvider {
|
||||
},
|
||||
isPending: (task) =>
|
||||
task.status === 'submitted' || task.status === 'processing',
|
||||
meter,
|
||||
hold,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -37,12 +37,14 @@ import {
|
||||
type MockInstance,
|
||||
} from 'vitest';
|
||||
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../../integrationTestUtil.js';
|
||||
import { withAiCostFactor } from '../../../util/aiCostFactor.js';
|
||||
import type { IGenerateVideoParams } from '../../types.js';
|
||||
import { VIDEO_POLL_WINDOW_MS } from '../polling.js';
|
||||
import { MiniMaxVideoProvider } from './MiniMaxVideoProvider.js';
|
||||
import { MINIMAX_VIDEO_GENERATION_MODELS } from './models.js';
|
||||
|
||||
@@ -648,6 +650,7 @@ describe('MiniMaxVideoProvider failures', () => {
|
||||
h3Task({
|
||||
status,
|
||||
content: undefined,
|
||||
usage: undefined,
|
||||
error: { code: '1000', message: 'unknown error' },
|
||||
}),
|
||||
);
|
||||
@@ -659,11 +662,26 @@ describe('MiniMaxVideoProvider failures', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it('passes through the seconds a failed H3 task reports', async () => {
|
||||
mockH3Flow(
|
||||
h3Task({
|
||||
status: 'failed',
|
||||
content: undefined,
|
||||
usage: { output_seconds: 3 },
|
||||
error: { code: '1000', message: 'unknown error' },
|
||||
}),
|
||||
);
|
||||
await expect(generate({})).rejects.toMatchObject({ statusCode: 502 });
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
expect(incrementUsageSpy.mock.calls[0]![2]).toBe(3);
|
||||
});
|
||||
|
||||
it('maps an H3 task failed by the content filter to moderation_flagged', async () => {
|
||||
mockH3Flow(
|
||||
h3Task({
|
||||
status: 'failed',
|
||||
content: undefined,
|
||||
usage: undefined,
|
||||
error: {
|
||||
code: '1026',
|
||||
message: 'video description contains sensitive content',
|
||||
@@ -854,13 +872,14 @@ describe('MiniMaxVideoProvider failures', () => {
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails when the finished H3 task has no video', async () => {
|
||||
// MiniMax bills a task it reports as succeeded, video or not.
|
||||
it('fails, still metered, when the finished H3 task has no video', async () => {
|
||||
mockH3Flow(h3Task({ content: {} }));
|
||||
await expect(generate({})).rejects.toThrow(/video URL/);
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('fails when the Hailuo file has no download URL', async () => {
|
||||
it('fails, still metered, when the Hailuo file has no download URL', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
jsonResponse({ task_id: 'task-1', base_resp: ok }),
|
||||
);
|
||||
@@ -871,6 +890,119 @@ describe('MiniMaxVideoProvider failures', () => {
|
||||
await expect(generate({ model: 'minimax-hailuo-2.3' })).rejects.toThrow(
|
||||
/video URL/,
|
||||
);
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('retries a Hailuo file lookup that hits a transient error', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
jsonResponse({ task_id: 'task-1', base_resp: ok }),
|
||||
);
|
||||
fetchSpy.mockResolvedValueOnce(jsonResponse(hailuoTask('Success')));
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
jsonResponse({ error: { message: 'upstream busy' } }, 503),
|
||||
);
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
file: { download_url: 'https://minimax.example/hailuo.mp4' },
|
||||
base_resp: ok,
|
||||
}),
|
||||
);
|
||||
await expect(generate({ model: 'minimax-hailuo-2.3' })).resolves.toBe(
|
||||
'https://minimax.example/hailuo.mp4',
|
||||
);
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MiniMaxVideoProvider after the caller leaves', () => {
|
||||
const disconnected = (params: Partial<IGenerateVideoParams>) =>
|
||||
withTestActor(() => {
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return makeProvider().generate({
|
||||
prompt: 'a ball rolls',
|
||||
...params,
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Moves the clock past the poll window on the first status poll; later
|
||||
* polls return `late()`.
|
||||
*/
|
||||
const expireOnPoll = (pending: unknown, late: () => unknown) => {
|
||||
const realNow = Date.now.bind(Date);
|
||||
let skew = 0;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => realNow() + skew);
|
||||
let polls = 0;
|
||||
fetchSpy.mockImplementation(async () => {
|
||||
skew = VIDEO_POLL_WINDOW_MS + 1;
|
||||
return jsonResponse(polls++ === 0 ? pending : late());
|
||||
});
|
||||
};
|
||||
|
||||
const timedOut = async (params: Partial<IGenerateVideoParams>) => {
|
||||
try {
|
||||
await generate(params);
|
||||
} catch (e) {
|
||||
return e;
|
||||
}
|
||||
throw new Error('expected the call to reject');
|
||||
};
|
||||
|
||||
it('keeps polling an H3 task after a disconnect and meters it', async () => {
|
||||
mockH3Flow(h3Task({ status: 'running' }), h3Task());
|
||||
await expect(
|
||||
disconnected({ model: 'minimax-h3', seconds: 5 }),
|
||||
).resolves.toBe('https://minimax.example/out.mp4');
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
expect(incrementUsageSpy.mock.calls[0]![2]).toBe(5);
|
||||
});
|
||||
|
||||
it('keeps polling a Hailuo task after a disconnect and meters it', async () => {
|
||||
mockHailuoFlow(hailuoTask('Processing'), hailuoTask('Success'));
|
||||
await expect(
|
||||
disconnected({ model: 'minimax-hailuo-2.3', seconds: 6 }),
|
||||
).resolves.toBe('https://minimax.example/hailuo.mp4');
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('meters the H3 seconds reported after the window', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(jsonResponse({ task_id: 'task-1' }));
|
||||
let finished = false;
|
||||
const running = { task: h3Task({ status: 'running' }) };
|
||||
expireOnPoll(running, () =>
|
||||
finished
|
||||
? { task: h3Task({ usage: { output_seconds: 4 } }) }
|
||||
: running,
|
||||
);
|
||||
expect(
|
||||
await timedOut({ model: 'minimax-h3', seconds: 5 }),
|
||||
).toMatchObject({ statusCode: 504, legacyCode: 'upstream_timeout' });
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
|
||||
finished = true;
|
||||
await vi.waitFor(() =>
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1),
|
||||
);
|
||||
const [, usageType, amount, cost] = incrementUsageSpy.mock.calls[0]!;
|
||||
expect(usageType).toBe(
|
||||
'minimax-video-generation:minimax-h3:per-second:768p',
|
||||
);
|
||||
expect(amount).toBe(4);
|
||||
expect(cost).toBeCloseTo(
|
||||
4 * rate('minimax-h3', 'per-second:768p') * 1_000_000,
|
||||
);
|
||||
});
|
||||
|
||||
it('meters nothing for a Hailuo clip that fails after the window', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
jsonResponse({ task_id: 'task-1', base_resp: ok }),
|
||||
);
|
||||
expireOnPoll(hailuoTask('Processing'), () => hailuoTask('Fail'));
|
||||
expect(
|
||||
await timedOut({ model: 'minimax-hailuo-2.3', seconds: 6 }),
|
||||
).toMatchObject({ statusCode: 504 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -25,11 +25,16 @@ import type { CreditHold } from '../../../../services/metering/types.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import { readBoundedBody } from '../../../util/fileInput.js';
|
||||
import { toUrlOrDataUri } from '../../../util/imageInput.js';
|
||||
import { sanitizeUpstreamMessage } from '../../../util/upstreamErrors.js';
|
||||
import {
|
||||
isTransientUpstreamError,
|
||||
sanitizeUpstreamMessage,
|
||||
} from '../../../util/upstreamErrors.js';
|
||||
import { capSecondsToRemainingCredits } from '../../creditCap.js';
|
||||
import type { IGenerateVideoParams, IVideoModel } from '../../types.js';
|
||||
import { VideoProvider } from '../VideoProvider.js';
|
||||
import {
|
||||
abortableDelay,
|
||||
JobHold,
|
||||
pollUntilSettled,
|
||||
VIDEO_HOLD_TTL_MS,
|
||||
videoJobFailure,
|
||||
@@ -46,6 +51,7 @@ const DEFAULT_MODEL = 'minimax-h3';
|
||||
const DEFAULT_POLL_INTERVAL_MS = 5_000;
|
||||
const REQUEST_TIMEOUT_MS = 60_000;
|
||||
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||
const FILE_RETRIEVE_ATTEMPTS = 3;
|
||||
const PROVIDER = 'minimax';
|
||||
|
||||
const ASPECT_RATIOS = ['21:9', '16:9', '4:3', '1:1', '3:4', '9:16'];
|
||||
@@ -323,6 +329,7 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
allowedSeconds: model.durationSeconds,
|
||||
});
|
||||
|
||||
const jobHold = new JobHold(hold);
|
||||
try {
|
||||
const content: Array<Record<string, unknown>> = [
|
||||
{ type: 'text', text: input.prompt },
|
||||
@@ -384,6 +391,31 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
},
|
||||
isPending: (task) =>
|
||||
task.status === 'queued' || task.status === 'running',
|
||||
// MiniMax bills the seconds of output the task reports. A
|
||||
// finished task without them is billed its clip length, or
|
||||
// the one it was asked for; a failed one only what it reports.
|
||||
meter: async (task, settled) => {
|
||||
const succeeded = task?.status === 'succeeded';
|
||||
const outputSeconds =
|
||||
task?.usage?.output_seconds ??
|
||||
(succeeded ? task?.duration : undefined);
|
||||
const billedSeconds =
|
||||
typeof outputSeconds === 'number' &&
|
||||
Number.isFinite(outputSeconds) &&
|
||||
outputSeconds > 0
|
||||
? Math.max(1, Math.round(outputSeconds))
|
||||
: succeeded || !settled
|
||||
? cappedSeconds
|
||||
: 0;
|
||||
if (!billedSeconds) return;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
billedSeconds,
|
||||
Math.round(billedSeconds * centsPerSecond * 1_000_000),
|
||||
);
|
||||
},
|
||||
hold: jobHold,
|
||||
});
|
||||
if (task.status !== 'succeeded') {
|
||||
throw videoJobFailure(
|
||||
@@ -396,24 +428,9 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
if (typeof url !== 'string' || !url.trim()) {
|
||||
throw new Error('MiniMax response did not include a video URL');
|
||||
}
|
||||
|
||||
// MiniMax bills the seconds of output the task reports.
|
||||
const outputSeconds = task.usage?.output_seconds ?? task.duration;
|
||||
const billedSeconds =
|
||||
typeof outputSeconds === 'number' &&
|
||||
Number.isFinite(outputSeconds) &&
|
||||
outputSeconds > 0
|
||||
? Math.max(1, Math.round(outputSeconds))
|
||||
: cappedSeconds;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
billedSeconds,
|
||||
Math.round(billedSeconds * centsPerSecond * 1_000_000),
|
||||
);
|
||||
return url;
|
||||
} finally {
|
||||
await hold.release();
|
||||
await jobHold.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,6 +470,7 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
|
||||
const jobHold = new JobHold(hold);
|
||||
try {
|
||||
const body: Record<string, unknown> = {
|
||||
model: spec.apiModel,
|
||||
@@ -484,6 +502,17 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
task.status === 'Preparing' ||
|
||||
task.status === 'Queueing' ||
|
||||
task.status === 'Processing',
|
||||
// Billed per clip; a failed one isn't.
|
||||
meter: async (task, settled) => {
|
||||
if (settled && task?.status !== 'Success') return;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
1,
|
||||
microCents,
|
||||
);
|
||||
},
|
||||
hold: jobHold,
|
||||
});
|
||||
if (task.status !== 'Success' || !task.file_id) {
|
||||
throw videoJobFailure(
|
||||
@@ -492,27 +521,17 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
);
|
||||
}
|
||||
// The clip is a file in MiniMax's store; its download link is
|
||||
// minted on request and works for an hour.
|
||||
const { file } = await this.#request<{
|
||||
file?: { download_url?: string };
|
||||
}>(
|
||||
'GET',
|
||||
`/v1/files/retrieve?file_id=${encodeURIComponent(task.file_id)}`,
|
||||
);
|
||||
// minted on request and works for an hour. The clip is already
|
||||
// billed, so a blip here is retried rather than failing it.
|
||||
const { file } = await this.#retrieveFile(task.file_id);
|
||||
const url = file?.download_url;
|
||||
if (typeof url !== 'string' || !url.trim()) {
|
||||
throw new Error('MiniMax response did not include a video URL');
|
||||
}
|
||||
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
1,
|
||||
microCents,
|
||||
);
|
||||
return url;
|
||||
} finally {
|
||||
await hold.release();
|
||||
await jobHold.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -569,6 +588,26 @@ export class MiniMaxVideoProvider extends VideoProvider {
|
||||
}
|
||||
}
|
||||
|
||||
async #retrieveFile(
|
||||
fileId: string,
|
||||
): Promise<{ file?: { download_url?: string } }> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
return await this.#request(
|
||||
'GET',
|
||||
`/v1/files/retrieve?file_id=${encodeURIComponent(fileId)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
if (
|
||||
attempt >= FILE_RETRIEVE_ATTEMPTS ||
|
||||
!isTransientUpstreamError(err)
|
||||
)
|
||||
throw err;
|
||||
await abortableDelay(this.#pollIntervalMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #request<T>(
|
||||
method: 'GET' | 'POST',
|
||||
path: string,
|
||||
|
||||
@@ -21,9 +21,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { Context, runWithContext } from '../../../core/context.js';
|
||||
import { HttpError } from '../../../core/http/HttpError.js';
|
||||
import {
|
||||
JobHold,
|
||||
LATE_JOB_WINDOW_MS,
|
||||
VIDEO_POLL_WINDOW_MS,
|
||||
abortableDelay,
|
||||
clientAbortedError,
|
||||
pollTimeoutError,
|
||||
pollUntilSettled,
|
||||
videoJobFailure,
|
||||
@@ -49,16 +50,6 @@ describe('pollTimeoutError', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('clientAbortedError', () => {
|
||||
it('builds a 400 client_aborted that names the provider', () => {
|
||||
expect(clientAbortedError('gemini')).toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'client_aborted',
|
||||
fields: { provider: 'gemini' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('videoJobFailure', () => {
|
||||
it('maps content-filter wording to 400 moderation_flagged', () => {
|
||||
expect(
|
||||
@@ -164,9 +155,19 @@ describe('pollUntilSettled', () => {
|
||||
const pending: Job = { status: 'pending' };
|
||||
const done: Job = { status: 'done' };
|
||||
|
||||
const fakeHold = () => ({
|
||||
release: vi.fn(() => Promise.resolve()),
|
||||
extend: vi.fn(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const poll = (
|
||||
fetch: (previous: Job | undefined) => Promise<Job>,
|
||||
extra: { initial?: Job; windowMs?: number } = {},
|
||||
extra: {
|
||||
initial?: Job;
|
||||
windowMs?: number;
|
||||
meter?: (job: Job | undefined, settled: boolean) => Promise<void>;
|
||||
hold?: JobHold;
|
||||
} = {},
|
||||
) =>
|
||||
runWithContext({}, () =>
|
||||
pollUntilSettled<Job>({
|
||||
@@ -175,10 +176,23 @@ describe('pollUntilSettled', () => {
|
||||
intervalMs: 5_000,
|
||||
fetch,
|
||||
isPending: (job) => job.status === 'pending',
|
||||
meter: () => Promise.resolve(),
|
||||
hold: new JobHold(fakeHold()),
|
||||
...extra,
|
||||
}),
|
||||
);
|
||||
|
||||
/** Settles a poll that is expected to reject, for later inspection. */
|
||||
const rejectionOf = (run: Promise<unknown>) =>
|
||||
(async () => {
|
||||
try {
|
||||
await run;
|
||||
} catch (e) {
|
||||
return e;
|
||||
}
|
||||
throw new Error('expected the poll to reject');
|
||||
})();
|
||||
|
||||
it('fetches immediately, then on the interval, until the job settles', async () => {
|
||||
vi.useFakeTimers();
|
||||
const fetch = vi
|
||||
@@ -197,6 +211,13 @@ describe('pollUntilSettled', () => {
|
||||
expect(fetch).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('meters the settled job once before returning it', async () => {
|
||||
const meter = vi.fn(() => Promise.resolve());
|
||||
expect(await poll(() => Promise.resolve(done), { meter })).toBe(done);
|
||||
expect(meter).toHaveBeenCalledOnce();
|
||||
expect(meter).toHaveBeenCalledWith(done, true);
|
||||
});
|
||||
|
||||
it('waits one interval before re-fetching a pending initial state', async () => {
|
||||
vi.useFakeTimers();
|
||||
const fetch = vi
|
||||
@@ -228,13 +249,27 @@ describe('pollUntilSettled', () => {
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('rethrows a poll failure that is the provider\'s verdict', async () => {
|
||||
it("rethrows a poll failure that is the provider's verdict, metering the job as unsettled", async () => {
|
||||
const notFound = Object.assign(new Error('no such job'), {
|
||||
status: 404,
|
||||
});
|
||||
await expect(poll(() => Promise.reject(notFound))).rejects.toBe(
|
||||
notFound,
|
||||
);
|
||||
const meter = vi.fn(() => Promise.resolve());
|
||||
await expect(
|
||||
poll(() => Promise.reject(notFound), { meter }),
|
||||
).rejects.toBe(notFound);
|
||||
expect(meter).toHaveBeenCalledOnce();
|
||||
expect(meter).toHaveBeenCalledWith(undefined, false);
|
||||
});
|
||||
|
||||
it("meters nothing when a poll is refused as the provider's verdict on the job", async () => {
|
||||
const refused = Object.assign(new Error('sensitive content'), {
|
||||
status: 400,
|
||||
});
|
||||
const meter = vi.fn(() => Promise.resolve());
|
||||
await expect(
|
||||
poll(() => Promise.reject(refused), { meter }),
|
||||
).rejects.toBe(refused);
|
||||
expect(meter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('gives up after the window with a 504 carrying the last poll error', async () => {
|
||||
@@ -244,7 +279,7 @@ describe('pollUntilSettled', () => {
|
||||
.fn<(p: Job | undefined) => Promise<Job>>()
|
||||
.mockRejectedValue(flaky);
|
||||
|
||||
const rejection = poll(fetch).catch((e: unknown) => e);
|
||||
const rejection = rejectionOf(poll(fetch));
|
||||
await vi.advanceTimersByTimeAsync(VIDEO_POLL_WINDOW_MS + 5_000);
|
||||
|
||||
expect(await rejection).toMatchObject({
|
||||
@@ -256,22 +291,50 @@ describe('pollUntilSettled', () => {
|
||||
|
||||
it('honours a shorter window when one is given', async () => {
|
||||
vi.useFakeTimers();
|
||||
const rejection = poll(() => Promise.resolve(pending), {
|
||||
windowMs: 20_000,
|
||||
}).catch((e: unknown) => e);
|
||||
const rejection = rejectionOf(
|
||||
poll(() => Promise.resolve(pending), { windowMs: 20_000 }),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(25_000);
|
||||
|
||||
expect(await rejection).toMatchObject({ statusCode: 504 });
|
||||
});
|
||||
|
||||
it('stops as soon as the request context signals a client abort', async () => {
|
||||
it('names the media in the timeout message', async () => {
|
||||
vi.useFakeTimers();
|
||||
const rejection = rejectionOf(
|
||||
runWithContext({}, () =>
|
||||
pollUntilSettled<Job>({
|
||||
provider: 'kling',
|
||||
providerLabel: 'Kling',
|
||||
noun: 'image',
|
||||
intervalMs: 5_000,
|
||||
windowMs: 0,
|
||||
initial: pending,
|
||||
fetch: () => Promise.resolve(pending),
|
||||
isPending: (job) => job.status === 'pending',
|
||||
meter: () => Promise.resolve(),
|
||||
hold: new JobHold(fakeHold()),
|
||||
}),
|
||||
),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
|
||||
expect(await rejection).toMatchObject({
|
||||
message: 'Timed out waiting for Kling image generation to complete',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps polling after a client abort, so the settled job is metered', async () => {
|
||||
vi.useFakeTimers();
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
const meter = vi.fn(() => Promise.resolve());
|
||||
const fetch = vi
|
||||
.fn<(p: Job | undefined) => Promise<Job>>()
|
||||
.mockResolvedValue(pending);
|
||||
.mockResolvedValueOnce(pending)
|
||||
.mockResolvedValueOnce(done);
|
||||
|
||||
const rejection = runWithContext({}, () => {
|
||||
const result = runWithContext({}, () => {
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return pollUntilSettled<Job>({
|
||||
provider: 'gemini',
|
||||
@@ -279,19 +342,131 @@ describe('pollUntilSettled', () => {
|
||||
intervalMs: 10_000,
|
||||
fetch,
|
||||
isPending: (job) => job.status === 'pending',
|
||||
meter,
|
||||
hold: new JobHold(fakeHold()),
|
||||
});
|
||||
}).catch((e: unknown) => e);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
abort.abort();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(await rejection).toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'client_aborted',
|
||||
fields: { provider: 'gemini' },
|
||||
});
|
||||
// The abort cut the second wait short; no third poll went out.
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
|
||||
expect(await result).toBe(done);
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
expect(meter).toHaveBeenCalledWith(done, true);
|
||||
});
|
||||
|
||||
describe('a job that outlives the window', () => {
|
||||
/** Times out on the first poll, then lets the test drive the job. */
|
||||
const lateJob = (fetch: (p: Job | undefined) => Promise<Job>) => {
|
||||
const hold = fakeHold();
|
||||
const jobHold = new JobHold(hold);
|
||||
const meter = vi.fn(() => Promise.resolve());
|
||||
const kept: Promise<unknown>[] = [];
|
||||
const rejection = rejectionOf(
|
||||
runWithContext({}, () => {
|
||||
Context.set('keepDriverSlot', (work) => kept.push(work));
|
||||
return pollUntilSettled<Job>({
|
||||
provider: 'together',
|
||||
providerLabel: 'Together AI',
|
||||
intervalMs: 5_000,
|
||||
windowMs: 0,
|
||||
initial: pending,
|
||||
fetch,
|
||||
isPending: (job) => job.status === 'pending',
|
||||
meter,
|
||||
hold: jobHold,
|
||||
});
|
||||
}),
|
||||
);
|
||||
return { hold, jobHold, meter, kept, rejection };
|
||||
};
|
||||
|
||||
it('answers 504 and meters the job once it settles', async () => {
|
||||
vi.useFakeTimers();
|
||||
const fetch = vi
|
||||
.fn<(p: Job | undefined) => Promise<Job>>()
|
||||
.mockResolvedValueOnce(pending)
|
||||
.mockResolvedValueOnce(pending)
|
||||
.mockResolvedValueOnce(done);
|
||||
const { hold, jobHold, meter, kept, rejection } = lateJob(fetch);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
expect(await rejection).toMatchObject({ statusCode: 504 });
|
||||
// The request's own release leaves the hold to the late job.
|
||||
await jobHold.release();
|
||||
expect(hold.release).not.toHaveBeenCalled();
|
||||
expect(meter).not.toHaveBeenCalled();
|
||||
expect(kept).toHaveLength(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
await kept[0];
|
||||
expect(meter).toHaveBeenCalledOnce();
|
||||
expect(meter).toHaveBeenCalledWith(done, true);
|
||||
expect(hold.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('renews the hold while it follows the job', async () => {
|
||||
vi.useFakeTimers();
|
||||
const { hold, rejection } = lateJob(() => Promise.resolve(pending));
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await rejection;
|
||||
|
||||
expect(hold.extend).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(11 * 60 * 1000);
|
||||
expect(hold.extend).toHaveBeenCalledTimes(3);
|
||||
expect(hold.release).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('meters a job still pending at the late deadline as unsettled', async () => {
|
||||
vi.useFakeTimers();
|
||||
const { hold, meter, kept, rejection } = lateJob(() =>
|
||||
Promise.resolve(pending),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await rejection;
|
||||
|
||||
vi.setSystemTime(Date.now() + LATE_JOB_WINDOW_MS);
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
await kept[0];
|
||||
expect(meter).toHaveBeenCalledOnce();
|
||||
expect(meter).toHaveBeenCalledWith(pending, false);
|
||||
expect(hold.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('meters nothing when a late poll is refused as a verdict', async () => {
|
||||
vi.useFakeTimers();
|
||||
const fetch = vi
|
||||
.fn<(p: Job | undefined) => Promise<Job>>()
|
||||
.mockResolvedValueOnce(pending)
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('sensitive content'), {
|
||||
status: 400,
|
||||
}),
|
||||
);
|
||||
const { hold, meter, kept, rejection } = lateJob(fetch);
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await rejection;
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await kept[0];
|
||||
expect(meter).not.toHaveBeenCalled();
|
||||
expect(hold.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('meters as unsettled when the provider loses track of the job', async () => {
|
||||
vi.useFakeTimers();
|
||||
const fetch = vi
|
||||
.fn<(p: Job | undefined) => Promise<Job>>()
|
||||
.mockResolvedValueOnce(pending)
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('no such job'), { status: 404 }),
|
||||
);
|
||||
const { hold, meter, kept, rejection } = lateJob(fetch);
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await rejection;
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await kept[0];
|
||||
expect(meter).toHaveBeenCalledWith(pending, false);
|
||||
expect(hold.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -19,10 +19,12 @@
|
||||
|
||||
import { Context } from '../../../core/context.js';
|
||||
import { HttpError } from '../../../core/http/HttpError.js';
|
||||
import type { CreditHold } from '../../../services/metering/types.js';
|
||||
import {
|
||||
CONTENT_FILTER_PATTERN,
|
||||
isTransientUpstreamError,
|
||||
sanitizeUpstreamMessage,
|
||||
upstreamErrorStatus,
|
||||
} from '../../util/upstreamErrors.js';
|
||||
|
||||
/**
|
||||
@@ -34,6 +36,13 @@ export const VIDEO_POLL_WINDOW_MS = 14 * 60 * 1000;
|
||||
/** A job's credit hold covers the poll window plus the create call. */
|
||||
export const VIDEO_HOLD_TTL_MS = VIDEO_POLL_WINDOW_MS + 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* How long a job that outlived the window is still followed after the caller
|
||||
* got its 504. One still pending after this is charged its estimate.
|
||||
*/
|
||||
export const LATE_JOB_WINDOW_MS = 60 * 60 * 1000;
|
||||
const LATE_HOLD_RENEW_MS = 5 * 60 * 1000;
|
||||
|
||||
// Provider wording for input the model does not accept, including Together's
|
||||
// camelCase codes (`invalidDuration`, `missingFrameImagesForImageToVideoModel`).
|
||||
const INVALID_INPUT_PATTERN =
|
||||
@@ -48,10 +57,11 @@ export const pollTimeoutError = (
|
||||
provider: string,
|
||||
providerLabel: string,
|
||||
cause?: unknown,
|
||||
noun: 'video' | 'image' = 'video',
|
||||
): HttpError =>
|
||||
new HttpError(
|
||||
504,
|
||||
`Timed out waiting for ${providerLabel} video generation to complete`,
|
||||
`Timed out waiting for ${providerLabel} ${noun} generation to complete`,
|
||||
{
|
||||
legacyCode: 'upstream_timeout',
|
||||
fields: { provider },
|
||||
@@ -59,13 +69,6 @@ export const pollTimeoutError = (
|
||||
},
|
||||
);
|
||||
|
||||
/** The caller hung up mid-job; nobody is left to bill or answer. */
|
||||
export const clientAbortedError = (provider: string): HttpError =>
|
||||
new HttpError(400, 'Client disconnected before video generation finished', {
|
||||
legacyCode: 'client_aborted',
|
||||
fields: { provider },
|
||||
});
|
||||
|
||||
/**
|
||||
* Classifies a job the provider reports as failed. A content-filter refusal is
|
||||
* the caller's to act on; a rejected parameter is theirs to fix; anything else
|
||||
@@ -115,11 +118,34 @@ export const abortableDelay = (
|
||||
signal?.addEventListener('abort', done, { once: true });
|
||||
});
|
||||
|
||||
/**
|
||||
* A job's credit hold. Released when the request ends, unless a job that
|
||||
* outlived the window took it over; that job releases it once it settles.
|
||||
*/
|
||||
export class JobHold {
|
||||
#hold: CreditHold;
|
||||
#taken = false;
|
||||
|
||||
constructor(hold: CreditHold) {
|
||||
this.#hold = hold;
|
||||
}
|
||||
|
||||
async release(): Promise<void> {
|
||||
if (!this.#taken) await this.#hold.release();
|
||||
}
|
||||
|
||||
take(): CreditHold {
|
||||
this.#taken = true;
|
||||
return this.#hold;
|
||||
}
|
||||
}
|
||||
|
||||
export interface PollOptions<T> {
|
||||
/** Goes into `fields.provider` on every error raised here. */
|
||||
provider: string;
|
||||
/** Human-readable name for the timeout message. */
|
||||
providerLabel: string;
|
||||
noun?: 'video' | 'image';
|
||||
intervalMs: number;
|
||||
windowMs?: number;
|
||||
/** The state the create call already returned, if it returned one. */
|
||||
@@ -127,43 +153,131 @@ export interface PollOptions<T> {
|
||||
/** Fetches the current job state; receives the last state seen. */
|
||||
fetch: (previous: T | undefined) => Promise<T>;
|
||||
isPending: (job: T) => boolean;
|
||||
/**
|
||||
* Records what the job cost once it settles, failed or not: the provider
|
||||
* bills what it bills. `settled` is false for a job given up on while still
|
||||
* pending, which is charged as the provider would charge a finished one.
|
||||
*/
|
||||
meter: (job: T | undefined, settled: boolean) => Promise<unknown>;
|
||||
/** Kept past the window for a job still running, until it settles. */
|
||||
hold: JobHold;
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls a provider job until it settles. A transient poll failure is a missed
|
||||
* poll, not a failed job; a client disconnect ends the wait before anything is
|
||||
* metered; the window closes with a 504 that carries the last poll error.
|
||||
* Polls a provider job until it settles, then meters it. A transient poll
|
||||
* failure is a missed poll, not a failed job. A client disconnect doesn't stop
|
||||
* it: the provider bills the job either way. When the window closes the caller
|
||||
* gets a 504 carrying the last poll error, and the job is followed in the
|
||||
* background, holding its credit and concurrency slot, until it settles.
|
||||
*/
|
||||
export const pollUntilSettled = async <T>(opts: PollOptions<T>): Promise<T> => {
|
||||
const windowMs = opts.windowMs ?? VIDEO_POLL_WINDOW_MS;
|
||||
const signal = Context.get('abortSignal');
|
||||
const start = Date.now();
|
||||
let job = opts.initial;
|
||||
let lastError: unknown;
|
||||
let firstFetch = job === undefined;
|
||||
|
||||
for (;;) {
|
||||
if (job !== undefined && !opts.isPending(job)) return job;
|
||||
if (signal?.aborted) throw clientAbortedError(opts.provider);
|
||||
if (job !== undefined && !opts.isPending(job)) {
|
||||
await opts.meter(job, true);
|
||||
return job;
|
||||
}
|
||||
if (Date.now() - start > windowMs) {
|
||||
const late = followLateJob(opts, job, opts.hold.take());
|
||||
Context.get('keepDriverSlot')?.(late);
|
||||
throw pollTimeoutError(
|
||||
opts.provider,
|
||||
opts.providerLabel,
|
||||
lastError,
|
||||
opts.noun,
|
||||
);
|
||||
}
|
||||
if (!firstFetch) {
|
||||
await abortableDelay(opts.intervalMs, signal);
|
||||
// The delay ends early on abort; do not spend a poll on it.
|
||||
if (signal?.aborted) throw clientAbortedError(opts.provider);
|
||||
}
|
||||
if (!firstFetch) await abortableDelay(opts.intervalMs);
|
||||
firstFetch = false;
|
||||
try {
|
||||
job = await opts.fetch(job);
|
||||
lastError = undefined;
|
||||
} catch (err) {
|
||||
if (!isTransientUpstreamError(err)) throw err;
|
||||
lastError = err;
|
||||
if (isTransientUpstreamError(err)) {
|
||||
lastError = err;
|
||||
continue;
|
||||
}
|
||||
if (!isJobVerdict(err)) {
|
||||
// Lost track of a job the provider still runs: charged as
|
||||
// one that never settled, the same as after the window.
|
||||
try {
|
||||
await opts.meter(job, false);
|
||||
} catch (meterErr) {
|
||||
console.error(
|
||||
`[${opts.provider}] metering a lost job failed:`,
|
||||
meterErr,
|
||||
);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* A poll rejected as a client error is the provider's verdict on the job (a
|
||||
* content filter, a rejected input). Auth and not-found errors, and errors with
|
||||
* no status, mean we lost track of a job the provider still runs.
|
||||
*/
|
||||
const isJobVerdict = (err: unknown): boolean => {
|
||||
const status = upstreamErrorStatus(err);
|
||||
return (
|
||||
status !== undefined &&
|
||||
status >= 400 &&
|
||||
status < 500 &&
|
||||
![401, 403, 404].includes(status)
|
||||
);
|
||||
};
|
||||
|
||||
/** Polls a job past its window and meters it once it settles or is given up. */
|
||||
const followLateJob = async <T>(
|
||||
opts: PollOptions<T>,
|
||||
initial: T | undefined,
|
||||
hold: CreditHold,
|
||||
): Promise<void> => {
|
||||
const extend = async () => {
|
||||
try {
|
||||
await hold.extend?.();
|
||||
} catch {
|
||||
// The next tick retries.
|
||||
}
|
||||
};
|
||||
// The hold is near the end of its TTL by now.
|
||||
void extend();
|
||||
const renew = setInterval(extend, LATE_HOLD_RENEW_MS);
|
||||
renew.unref?.();
|
||||
let job = initial;
|
||||
try {
|
||||
const deadline = Date.now() + LATE_JOB_WINDOW_MS;
|
||||
while (job === undefined || opts.isPending(job)) {
|
||||
if (Date.now() > deadline) break;
|
||||
await abortableDelay(opts.intervalMs);
|
||||
try {
|
||||
job = await opts.fetch(job);
|
||||
} catch (err) {
|
||||
if (isTransientUpstreamError(err)) continue;
|
||||
// A refusal is a failure the provider doesn't bill.
|
||||
if (isJobVerdict(err)) return;
|
||||
break;
|
||||
}
|
||||
}
|
||||
await opts.meter(job, job !== undefined && !opts.isPending(job));
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[${opts.provider}] metering a job that outlived its window failed:`,
|
||||
err,
|
||||
);
|
||||
} finally {
|
||||
clearInterval(renew);
|
||||
try {
|
||||
await hold.release();
|
||||
} catch {
|
||||
// Left to its TTL.
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -453,31 +453,76 @@ describe('TogetherVideoProvider.generate polling', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('gives up after the wait window as HttpError 504 upstream_timeout, without metering', async () => {
|
||||
/** Starts a job that is still running when the wait window closes. */
|
||||
const outliveWindow = async () => {
|
||||
const provider = makeProvider();
|
||||
videosCreateMock.mockResolvedValueOnce({ id: 'job-slow' });
|
||||
videosRetrieveMock.mockResolvedValue({
|
||||
id: 'job-slow',
|
||||
status: 'in_progress',
|
||||
});
|
||||
|
||||
const rejection = (async () => {
|
||||
try {
|
||||
await withTestActor(() =>
|
||||
provider.generate({
|
||||
prompt: 'hi',
|
||||
model: 'togetherai:minimax/video-01-director',
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
return e;
|
||||
}
|
||||
})();
|
||||
|
||||
// Ten-minute wait window, polled every 5s.
|
||||
await vi.advanceTimersByTimeAsync(VIDEO_POLL_WINDOW_MS + 5_000);
|
||||
|
||||
expect(await rejection).toMatchObject({
|
||||
statusCode: 504,
|
||||
legacyCode: 'upstream_timeout',
|
||||
message:
|
||||
'Timed out waiting for Together AI video generation to complete',
|
||||
fields: { provider: 'together' },
|
||||
});
|
||||
// Together still renders and bills the job past the window.
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
};
|
||||
|
||||
it('passes through what Together charged for a job that fails after the window', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const provider = makeProvider();
|
||||
videosCreateMock.mockResolvedValueOnce({ id: 'job-slow' });
|
||||
await outliveWindow();
|
||||
videosRetrieveMock.mockResolvedValue({
|
||||
id: 'job-slow',
|
||||
status: 'in_progress',
|
||||
status: 'failed',
|
||||
outputs: { cost: 0.5 },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
|
||||
const rejection = withTestActor(() =>
|
||||
provider.generate({ prompt: 'hi' }),
|
||||
).catch((e: unknown) => e);
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
const [, usageType, count, cost] = incrementUsageSpy.mock.calls[0]!;
|
||||
expect(usageType).toBe('together-video:minimax/video-01-director');
|
||||
expect(count).toBe(1);
|
||||
expect(cost).toBe(50 * 1_000_000);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
// Ten-minute wait window, polled every 5s.
|
||||
await vi.advanceTimersByTimeAsync(VIDEO_POLL_WINDOW_MS + 5_000);
|
||||
it('charges the held estimate when Together loses track of a late job', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
await outliveWindow();
|
||||
videosRetrieveMock.mockRejectedValue(
|
||||
Object.assign(new Error('no such job'), { status: 404 }),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
|
||||
expect(await rejection).toMatchObject({
|
||||
statusCode: 504,
|
||||
legacyCode: 'upstream_timeout',
|
||||
message:
|
||||
'Timed out waiting for Together AI video generation to complete',
|
||||
fields: { provider: 'together' },
|
||||
});
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
const [, , count, cost] = incrementUsageSpy.mock.calls[0]!;
|
||||
expect(count).toBe(1);
|
||||
expect(cost).toBe(28 * 1_000_000);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
@@ -510,33 +555,35 @@ describe('TogetherVideoProvider.generate polling', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('stops polling and skips metering when the client disconnects', async () => {
|
||||
it('keeps polling and meters the finished job after the client disconnects', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const provider = makeProvider();
|
||||
videosCreateMock.mockResolvedValueOnce({ id: 'job-gone' });
|
||||
videosRetrieveMock.mockResolvedValue({
|
||||
id: 'job-gone',
|
||||
status: 'in_progress',
|
||||
});
|
||||
videosRetrieveMock
|
||||
.mockResolvedValueOnce({
|
||||
id: 'job-gone',
|
||||
status: 'in_progress',
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
id: 'job-gone',
|
||||
status: 'completed',
|
||||
outputs: { video_url: 'https://together/gone.mp4' },
|
||||
});
|
||||
const abort = new AbortController();
|
||||
|
||||
const rejection = withTestActor(() => {
|
||||
const result = withTestActor(() => {
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return provider.generate({ prompt: 'hi' });
|
||||
}).catch((e: unknown) => e);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
abort.abort();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(await rejection).toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'client_aborted',
|
||||
fields: { provider: 'together' },
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
abort.abort();
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
|
||||
expect(await result).toBe('https://together/gone.mp4');
|
||||
expect(videosRetrieveMock).toHaveBeenCalledTimes(2);
|
||||
expect(incrementUsageSpy).not.toHaveBeenCalled();
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
@@ -951,20 +998,24 @@ describe('TogetherVideoProvider.generate credit holds', () => {
|
||||
expect(await outstanding(actor)).toBe(0);
|
||||
});
|
||||
|
||||
it('releases the hold when the client disconnects mid-job', async () => {
|
||||
it('holds through a disconnected job and releases it once metered', async () => {
|
||||
const provider = makeProvider();
|
||||
const actor = holdUser('abort');
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
let heldWhilePolling = -1;
|
||||
videosCreateMock.mockResolvedValueOnce({ id: 'job-gone' });
|
||||
videosRetrieveMock.mockImplementationOnce(async () => {
|
||||
heldWhilePolling = await outstanding(actor);
|
||||
return completed('job-gone');
|
||||
});
|
||||
|
||||
await expect(
|
||||
withTestActor(() => {
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return provider.generate({ prompt: 'hi', model: SEEDANCE });
|
||||
}, actor),
|
||||
).rejects.toMatchObject({ legacyCode: 'client_aborted' });
|
||||
expect(videosCreateMock).toHaveBeenCalledTimes(1);
|
||||
await withTestActor(() => {
|
||||
Context.set('abortSignal', abort.signal);
|
||||
return provider.generate({ prompt: 'hi', model: SEEDANCE });
|
||||
}, actor);
|
||||
expect(heldWhilePolling).toBeGreaterThan(0);
|
||||
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
|
||||
expect(await outstanding(actor)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -35,6 +35,7 @@ import type { IGenerateVideoParams, IVideoModel } from '../../types.js';
|
||||
import { capSecondsToRemainingCredits } from '../../creditCap.js';
|
||||
import { VideoProvider } from '../VideoProvider.js';
|
||||
import {
|
||||
JobHold,
|
||||
pollUntilSettled,
|
||||
VIDEO_HOLD_TTL_MS,
|
||||
videoJobFailure,
|
||||
@@ -242,6 +243,7 @@ export class TogetherVideoProvider extends VideoProvider {
|
||||
billedUnits = 1;
|
||||
}
|
||||
|
||||
const jobHold = new JobHold(hold);
|
||||
try {
|
||||
// Read only once credit is held; the hold is released if it fails.
|
||||
const sourceVideo = hasInputVideo
|
||||
@@ -337,8 +339,39 @@ export class TogetherVideoProvider extends VideoProvider {
|
||||
createPayload.media = { source_video: sourceVideo };
|
||||
}
|
||||
|
||||
// Together reports what it actually charged for the job; the
|
||||
// catalog rate above is only the pre-flight estimate. Without a
|
||||
// reported cost a finished job is charged the estimate, a failed
|
||||
// or cancelled one nothing.
|
||||
const meter = async (finalJob: any, settled: boolean) => {
|
||||
const reportedCost = finalJob?.outputs?.cost;
|
||||
const reported =
|
||||
typeof reportedCost === 'number' &&
|
||||
Number.isFinite(reportedCost) &&
|
||||
reportedCost >= 0
|
||||
? Math.round(reportedCost * 100 * 1_000_000)
|
||||
: undefined;
|
||||
const succeeded =
|
||||
finalJob?.status !== 'failed' &&
|
||||
finalJob?.status !== 'cancelled';
|
||||
const cost =
|
||||
reported ??
|
||||
(succeeded || !settled ? estimateMicroCents : 0);
|
||||
if (!cost) return;
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageKey,
|
||||
billedUnits,
|
||||
cost,
|
||||
);
|
||||
};
|
||||
|
||||
const job = await this.#client.videos.create(createPayload);
|
||||
const finalJob = await this.#pollUntilComplete(job.id);
|
||||
const finalJob = await this.#pollUntilComplete(
|
||||
job.id,
|
||||
meter,
|
||||
jobHold,
|
||||
);
|
||||
|
||||
if (finalJob.status === 'failed') {
|
||||
const errorMessage =
|
||||
@@ -363,23 +396,6 @@ export class TogetherVideoProvider extends VideoProvider {
|
||||
);
|
||||
}
|
||||
|
||||
// Together reports what it actually charged for the job; the catalog
|
||||
// rate above was only the pre-flight estimate.
|
||||
const reportedCost = finalJob?.outputs?.cost;
|
||||
const costMicroCents =
|
||||
typeof reportedCost === 'number' &&
|
||||
Number.isFinite(reportedCost) &&
|
||||
reportedCost >= 0
|
||||
? Math.round(reportedCost * 100 * 1_000_000)
|
||||
: estimateMicroCents;
|
||||
|
||||
await this.#meteringService.incrementUsage(
|
||||
actor,
|
||||
usageKey,
|
||||
billedUnits,
|
||||
costMicroCents,
|
||||
);
|
||||
|
||||
const videoUrl = finalJob?.outputs?.video_url;
|
||||
if (typeof videoUrl === 'string' && videoUrl.trim()) {
|
||||
return videoUrl;
|
||||
@@ -387,7 +403,7 @@ export class TogetherVideoProvider extends VideoProvider {
|
||||
|
||||
throw new Error('Together AI response did not include a video URL');
|
||||
} finally {
|
||||
await hold.release();
|
||||
await jobHold.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -415,7 +431,11 @@ export class TogetherVideoProvider extends VideoProvider {
|
||||
});
|
||||
}
|
||||
|
||||
async #pollUntilComplete(jobId: string): Promise<any> {
|
||||
async #pollUntilComplete(
|
||||
jobId: string,
|
||||
meter: (job: any, settled: boolean) => Promise<unknown>,
|
||||
hold: JobHold,
|
||||
): Promise<any> {
|
||||
// any here because sdk types are wrong https://docs.together.ai/docs/videos-overview -> "Job Status Reference"
|
||||
return await pollUntilSettled<any>({
|
||||
provider: 'together',
|
||||
@@ -424,6 +444,8 @@ export class TogetherVideoProvider extends VideoProvider {
|
||||
fetch: () => (this.#client as any).videos.retrieve(jobId),
|
||||
isPending: (job) =>
|
||||
job.status === 'queued' || job.status === 'in_progress',
|
||||
meter,
|
||||
hold,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -271,8 +271,11 @@ export class AppDriver extends PuterDriver {
|
||||
legacyCode: 'app_name_already_in_use',
|
||||
});
|
||||
}
|
||||
if (filetypes)
|
||||
if (filetypes) {
|
||||
await this.appStore.setFiletypeAssociations(app.id, filetypes);
|
||||
// The row was built before they were stored; views re-read them.
|
||||
delete app.filetype_associations;
|
||||
}
|
||||
|
||||
this.#emitAppChanged({ app, action: 'created' });
|
||||
|
||||
@@ -421,10 +424,10 @@ export class AppDriver extends PuterDriver {
|
||||
existing: app,
|
||||
});
|
||||
|
||||
// Puter-hosted index_url handling on update — same flow as create
|
||||
// but only when the index_url is actually changing. Self-app is
|
||||
// Puter-hosted index_url handling on update — same flow as create.
|
||||
// #validateInput already dropped an unchanged index_url. Self-app is
|
||||
// excluded from the conflict search via `excludeAppId`.
|
||||
if (fields.index_url && fields.index_url !== app.index_url) {
|
||||
if (fields.index_url) {
|
||||
await this.#ensurePuterSiteSubdomainIsOwned(
|
||||
fields.index_url,
|
||||
actor.user,
|
||||
@@ -486,6 +489,8 @@ export class AppDriver extends PuterDriver {
|
||||
}
|
||||
if (filetypes !== undefined) {
|
||||
await this.appStore.setFiletypeAssociations(app.id, filetypes);
|
||||
// A filetype-only update returns the cached row with the old ones.
|
||||
if (updated) delete updated.filetype_associations;
|
||||
}
|
||||
|
||||
this.#emitAppChanged({ app: updated, old_app: app, action: 'updated' });
|
||||
@@ -615,13 +620,19 @@ export class AppDriver extends PuterDriver {
|
||||
protocols: WEB_AND_EXTENSION_PROTOCOLS,
|
||||
});
|
||||
if (out.index_url) out.index_url = new URL(out.index_url).href;
|
||||
// Only enforce on new/changed values so rows that already
|
||||
// carry a reserved host (migration-seeded builtins) can still
|
||||
// have their other fields updated.
|
||||
// A value equal to the stored one after normalization is
|
||||
// unchanged: drop it so legacy spellings stay as stored and
|
||||
// skip the host checks (migration-seeded builtins carry
|
||||
// reserved hosts). Rewriting it would make a dormant
|
||||
// mixed-case/default-port row match again without checks.
|
||||
if (
|
||||
existing &&
|
||||
out.index_url !== undefined &&
|
||||
out.index_url !== existing?.index_url
|
||||
out.index_url ===
|
||||
(URL.parse(existing.index_url)?.href ?? existing.index_url)
|
||||
) {
|
||||
delete out.index_url;
|
||||
} else if (out.index_url !== undefined) {
|
||||
this.#assertIndexUrlHostAllowed(out.index_url);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { makeActor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
|
||||
let server: PuterServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer({
|
||||
domain: 'puter.localhost',
|
||||
static_hosting_domain: 'site.puter.localhost',
|
||||
protocol: 'http',
|
||||
} as unknown as IConfig);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
type AppsDriver = {
|
||||
update: (args: {
|
||||
id: { name: string };
|
||||
object: Record<string, unknown>;
|
||||
}) => Promise<Record<string, unknown>>;
|
||||
};
|
||||
|
||||
describe('saving app settings with an unchanged legacy index_url', () => {
|
||||
it('allows saving an unchanged URL after its site is gone', async () => {
|
||||
const username = `li-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const user = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
} as Parameters<typeof server.stores.user.create>[0]);
|
||||
const actor = makeActor({
|
||||
user: { ...user, email_confirmed: true } as never,
|
||||
});
|
||||
const sub = `li-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const legacyUrl = `http://${sub}.site.puter.localhost`;
|
||||
const name = `li-app-${Math.random().toString(36).slice(2, 8)}`;
|
||||
// Row as Dev Center wrote it before normalization: no trailing slash.
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[`app-${uuidv4()}`, name, 'Old title', legacyUrl, user.id],
|
||||
);
|
||||
const apps = server.drivers.apps as unknown as AppsDriver;
|
||||
|
||||
// The site it pointed at has since been deleted.
|
||||
expect(await server.stores.subdomain.getBySubdomain(sub)).toBeFalsy();
|
||||
|
||||
// Title-only save works.
|
||||
await runWithContext({ actor }, () =>
|
||||
apps.update({ id: { name }, object: { title: 'Title only' } }),
|
||||
);
|
||||
|
||||
// Dev Center's save sends the stored index_url back as it was.
|
||||
await expect(
|
||||
runWithContext({ actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: { title: 'New title', index_url: legacyUrl },
|
||||
}),
|
||||
),
|
||||
).resolves.toMatchObject({ title: 'New title' });
|
||||
await expect(
|
||||
runWithContext({ actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: {
|
||||
index_url: `http://different-${sub}.site.puter.localhost/`,
|
||||
},
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'subdomain_not_owned',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes a stored reserved host before comparing unchanged values', async () => {
|
||||
const user = await server.stores.user.create({
|
||||
username: `legacy-${uuidv4().slice(0, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
});
|
||||
const actor = makeActor({ user });
|
||||
const name = `legacy-${uuidv4()}`;
|
||||
const indexUrl = 'https://builtins.namespaces.puter.com';
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[`app-${uuidv4()}`, name, 'Builtin', indexUrl, user.id],
|
||||
);
|
||||
const apps = server.drivers.apps as unknown as AppsDriver;
|
||||
await expect(
|
||||
runWithContext({ actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: { index_url: indexUrl, title: 'Saved' },
|
||||
}),
|
||||
),
|
||||
).resolves.toMatchObject({ title: 'Saved' });
|
||||
await expect(
|
||||
runWithContext({ actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: { index_url: `${indexUrl}/different` },
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('leaves a legacy mixed-case row as stored when re-sent normalized', async () => {
|
||||
const mkUser = async () => {
|
||||
const username = `lc-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const user = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
} as Parameters<typeof server.stores.user.create>[0]);
|
||||
return {
|
||||
user,
|
||||
actor: makeActor({
|
||||
user: { ...user, email_confirmed: true } as never,
|
||||
}),
|
||||
};
|
||||
};
|
||||
const owner = await mkUser();
|
||||
const siteOwner = await mkUser();
|
||||
const sub = `lc${Math.random().toString(36).slice(2, 8)}`;
|
||||
const otherSub = `lc${Math.random().toString(36).slice(2, 8)}`;
|
||||
for (const s of [sub, otherSub]) {
|
||||
const home = await server.services.fs.mkdir(siteOwner.user.id, {
|
||||
path: `/${siteOwner.user.username}/${s}`,
|
||||
createMissingParents: true,
|
||||
});
|
||||
await server.stores.subdomain.create({
|
||||
userId: siteOwner.user.id,
|
||||
subdomain: s,
|
||||
rootDirId: home.id,
|
||||
});
|
||||
}
|
||||
// Pre-normalization row pointing at a site now owned by someone else.
|
||||
const legacyUrl = `http://${sub.toUpperCase()}.site.puter.localhost`;
|
||||
const name = `lc-app-${uuidv4().slice(0, 8)}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[`app-${uuidv4()}`, name, 'Old', legacyUrl, owner.user.id],
|
||||
);
|
||||
const apps = server.drivers.apps as unknown as AppsDriver;
|
||||
const readStored = async () => {
|
||||
const [row] = await server.clients.db.read(
|
||||
'SELECT index_url, title FROM apps WHERE name = ?',
|
||||
[name],
|
||||
);
|
||||
return row;
|
||||
};
|
||||
|
||||
await runWithContext({ actor: owner.actor }, () =>
|
||||
apps.update({ id: { name }, object: { title: 'Title only' } }),
|
||||
);
|
||||
expect(await readStored()).toMatchObject({
|
||||
index_url: legacyUrl,
|
||||
title: 'Title only',
|
||||
});
|
||||
|
||||
await runWithContext({ actor: owner.actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: {
|
||||
title: 'Resent',
|
||||
index_url: `http://${sub}.site.puter.localhost/`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(await readStored()).toMatchObject({
|
||||
index_url: legacyUrl,
|
||||
title: 'Resent',
|
||||
});
|
||||
|
||||
await expect(
|
||||
runWithContext({ actor: owner.actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: {
|
||||
index_url: `http://${otherSub}.site.puter.localhost/`,
|
||||
},
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'subdomain_not_owned',
|
||||
});
|
||||
expect((await readStored()).index_url).toBe(legacyUrl);
|
||||
});
|
||||
|
||||
it('leaves a legacy default-port reserved host as stored', async () => {
|
||||
const user = await server.stores.user.create({
|
||||
username: `legacy-${uuidv4().slice(0, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
});
|
||||
const actor = makeActor({ user });
|
||||
const name = `legacy-${uuidv4()}`;
|
||||
const indexUrl = 'https://PUTER.COM:443';
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[`app-${uuidv4()}`, name, 'Builtin', indexUrl, user.id],
|
||||
);
|
||||
const apps = server.drivers.apps as unknown as AppsDriver;
|
||||
await expect(
|
||||
runWithContext({ actor }, () =>
|
||||
apps.update({
|
||||
id: { name },
|
||||
object: { index_url: 'https://puter.com/', title: 'Saved' },
|
||||
}),
|
||||
),
|
||||
).resolves.toMatchObject({ title: 'Saved' });
|
||||
const [row] = await server.clients.db.read(
|
||||
'SELECT index_url FROM apps WHERE name = ?',
|
||||
[name],
|
||||
);
|
||||
expect(row.index_url).toBe(indexUrl);
|
||||
});
|
||||
});
|
||||
@@ -1055,6 +1055,34 @@ describe('AppDriver.create additional branches', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('returns the new filetype_associations from a filetype-only update of a cached app', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const created = await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName('ftu'),
|
||||
title: 't',
|
||||
index_url: uniqueIndexUrl(),
|
||||
filetype_associations: ['txt'],
|
||||
},
|
||||
}),
|
||||
);
|
||||
// Let the post-write window lapse so the row is cached again.
|
||||
for (const key of await server.clients.redis.keys('apps:*:deleted')) {
|
||||
await server.clients.redis.del(key);
|
||||
}
|
||||
await server.stores.app.getByUid(created.uid as string);
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
|
||||
const updated = await withActor(actor, () =>
|
||||
driver.update({
|
||||
uid: created.uid as string,
|
||||
object: { filetype_associations: ['csv'] },
|
||||
}),
|
||||
);
|
||||
expect(updated.filetype_associations).toEqual(['csv']);
|
||||
});
|
||||
|
||||
it('rejects too-long title with 400', async () => {
|
||||
const { actor } = await makeUser();
|
||||
await expect(
|
||||
@@ -2294,7 +2322,10 @@ describe('AppDriver.select query-count regression', () => {
|
||||
expect(large.itemCount).toBe(30);
|
||||
// Sanity check the classifier actually saw the shapes it's counting.
|
||||
expect(small.counts.indexUrlIn).toBeGreaterThan(0);
|
||||
expect(small.counts.subdomains).toBeGreaterThan(0);
|
||||
// Live and dangling backings both come from the warm subdomain cache
|
||||
// (negative entries included), so no subdomains read on either path.
|
||||
expect(small.counts.subdomains).toBe(0);
|
||||
expect(small.pread).toBe(0);
|
||||
|
||||
expect(large.counts).toEqual(small.counts);
|
||||
expect(large.pread).toBe(small.pread);
|
||||
|
||||
@@ -182,6 +182,14 @@ const appActorFor = (user: TestUser, app: { uid: string; id: number }) =>
|
||||
app: { uid: app.uid, id: app.id },
|
||||
});
|
||||
|
||||
/** What launching the app records: the user authorizing it. */
|
||||
const authorizeApp = (user: TestUser, app: { uid: string }) =>
|
||||
env.server.services.permission.grantUserAppPermission(
|
||||
user.actor,
|
||||
app.uid,
|
||||
'flag:app-is-authenticated',
|
||||
);
|
||||
|
||||
/** One `/drivers/call` to the KV driver, the way the SDK makes it. */
|
||||
const drive = async (
|
||||
token: string,
|
||||
@@ -922,6 +930,8 @@ describe('an app using a handle', () => {
|
||||
beforeAll(async () => {
|
||||
app = await makeApp();
|
||||
otherApp = await makeApp();
|
||||
await authorizeApp(guest, app);
|
||||
await authorizeApp(guest, otherApp);
|
||||
});
|
||||
|
||||
it('works in the app whose data it is, and in the holder`s own session', async () => {
|
||||
@@ -1025,6 +1035,7 @@ describe('a token using a handle', () => {
|
||||
);
|
||||
const inApp = await share(appPrefix, 'read', { appUid: app.uid });
|
||||
await share(appElsewhere, 'read', { appUid: app.uid });
|
||||
await authorizeApp(guest, app);
|
||||
const appActor = appActorFor(guest, app);
|
||||
|
||||
const carrying = await tokenActor(
|
||||
@@ -1407,3 +1418,250 @@ describe('what a holder is ever shown', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('a handle id', () => {
|
||||
/** Statements against the handle table while `call` runs. */
|
||||
const handleReads = async (call: () => Promise<unknown>) => {
|
||||
const read = vi.spyOn(env.server.clients.db, 'read');
|
||||
const pread = vi.spyOn(env.server.clients.db, 'pread');
|
||||
const touches = (spy: typeof read) =>
|
||||
spy.mock.calls.filter(([sql]) =>
|
||||
String(sql).includes('kv_share_handles'),
|
||||
).length;
|
||||
try {
|
||||
await call();
|
||||
} catch {
|
||||
// The caller asserts on the answer separately.
|
||||
}
|
||||
const counts = { read: touches(read), pread: touches(pread) };
|
||||
read.mockRestore();
|
||||
pread.mockRestore();
|
||||
return counts;
|
||||
};
|
||||
|
||||
it('is refused without a store read when its mac does not verify', async () => {
|
||||
const prefix = region('forged');
|
||||
await ownerSet(`${prefix}k`, 'v');
|
||||
const { handle } = await share(prefix, 'read');
|
||||
const [id, mac] = handle.split('.');
|
||||
const forged = `${id}.${mac[0] === 'A' ? 'B' : 'A'}${mac.slice(1)}`;
|
||||
|
||||
await expect(guestGet(forged, 'k')).rejects.toMatchObject({
|
||||
legacyCode: 'kv_handle_not_found',
|
||||
});
|
||||
expect(await handleReads(() => guestGet(forged, 'k'))).toEqual({
|
||||
read: 0,
|
||||
pread: 0,
|
||||
});
|
||||
await expect(
|
||||
events().subscribe(guest.actor, 'kv-forged-socket', {
|
||||
subject: `kv:${forged}:*`,
|
||||
}),
|
||||
).rejects.toMatchObject({ legacyCode: 'subject_does_not_exist' });
|
||||
expect(await guestGet(handle, 'k')).toBe('v');
|
||||
});
|
||||
|
||||
it('minted before ids were signed still works, and never falls back to the primary', async () => {
|
||||
const prefix = region('legacy-id');
|
||||
await ownerSet(`${prefix}k`, 'old');
|
||||
const permission = kvSharePermission(
|
||||
owner.uuid,
|
||||
GLOBAL,
|
||||
prefix,
|
||||
'read',
|
||||
);
|
||||
await env.server.services.permission.grantUserUserPermission(
|
||||
owner.actor,
|
||||
guest.username,
|
||||
permission,
|
||||
);
|
||||
const legacy = `kvh-${uuidv4()}`;
|
||||
await env.server.clients.db.insert('kv_share_handles', {
|
||||
handle: legacy,
|
||||
owner_user_id: owner.id,
|
||||
grantee_user_id: guest.id,
|
||||
app_uid: GLOBAL,
|
||||
key_prefix: prefix,
|
||||
permission,
|
||||
label: null,
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
revoked_at: null,
|
||||
});
|
||||
|
||||
expect(await guestGet(legacy, 'k')).toBe('old');
|
||||
expect(
|
||||
await handleReads(() => guestGet(`kvh-${uuidv4()}`, 'k')),
|
||||
).toEqual({ read: 1, pread: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('a handle used by an app its holder removed', () => {
|
||||
const revokeAll = async (appUid: string) => {
|
||||
const res = await fetch(
|
||||
new URL('/auth/revoke-user-app', env.apiOrigin),
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${guest.creds.token}`,
|
||||
},
|
||||
body: JSON.stringify({ app_uid: appUid, permission: '*' }),
|
||||
},
|
||||
);
|
||||
expect(res.status, await res.text()).toBe(200);
|
||||
};
|
||||
|
||||
it('is absent to the app until it is authorized again, and the holder keeps it', async () => {
|
||||
const app = await makeApp();
|
||||
const prefix = region('removed');
|
||||
await as(appActorFor(owner, app), (d) =>
|
||||
d.set({ key: `${prefix}k`, value: 'in app' }),
|
||||
);
|
||||
const { handle } = await share(prefix, 'read', { appUid: app.uid });
|
||||
await authorizeApp(guest, app);
|
||||
const appActor = appActorFor(guest, app);
|
||||
expect(await guestGet(handle, 'k', appActor)).toBe('in app');
|
||||
|
||||
await revokeAll(app.uid);
|
||||
await expect(guestGet(handle, 'k', appActor)).rejects.toMatchObject({
|
||||
legacyCode: 'kv_handle_not_found',
|
||||
});
|
||||
expect(await guestGet(handle, 'k')).toBe('in app');
|
||||
expect(await handles().getLive(handle)).not.toBeNull();
|
||||
|
||||
await authorizeApp(guest, app);
|
||||
expect(await guestGet(handle, 'k', appActor)).toBe('in app');
|
||||
});
|
||||
|
||||
it('stops delivering to the app`s live subscription', async () => {
|
||||
const app = await makeApp();
|
||||
const prefix = region('removed-sub');
|
||||
const { handle } = await share(prefix, 'read', { appUid: app.uid });
|
||||
await authorizeApp(guest, app);
|
||||
const socket = 'kv-removed-socket';
|
||||
const { sub } = await events().subscribe(
|
||||
appActorFor(guest, app),
|
||||
socket,
|
||||
{ subject: `kv:${handle}:*` },
|
||||
);
|
||||
const writes = (key: string) =>
|
||||
as(appActorFor(owner, app), (d) =>
|
||||
d.set({ key: `${prefix}${key}`, value: 1 }),
|
||||
);
|
||||
|
||||
delivered.length = 0;
|
||||
await writes('a');
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(delivered.some((one) => one.subId === sub.subId)).toBe(
|
||||
true,
|
||||
),
|
||||
{ timeout: EVENTS_COALESCE_WINDOW_MS * 12, interval: 25 },
|
||||
);
|
||||
|
||||
await revokeAll(app.uid);
|
||||
delivered.length = 0;
|
||||
await writes('b');
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, EVENTS_COALESCE_WINDOW_MS * 3),
|
||||
);
|
||||
expect(delivered.some((one) => one.subId === sub.subId)).toBe(false);
|
||||
await events().reapSocket(guest.id, socket);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listing keys through a watch handle', () => {
|
||||
it('returns keys relative to the handle, and refuses values', async () => {
|
||||
const prefix = region('watch-keys');
|
||||
await ownerSet(`${prefix}a`, 1);
|
||||
await ownerSet(`${prefix}b:c`, 2);
|
||||
const { handle } = await share(prefix, 'watch');
|
||||
|
||||
await expect(
|
||||
as(guest.actor, (d) =>
|
||||
d.list({ as: 'keys', includeTotal: true, ...via(handle) }),
|
||||
),
|
||||
).resolves.toEqual({ items: ['a', 'b:c'], total: 2 });
|
||||
for (const as_ of ['entries', 'values', undefined] as const)
|
||||
await expect(
|
||||
as(guest.actor, (d) => d.list({ as: as_, ...via(handle) })),
|
||||
).rejects.toMatchObject({ legacyCode: 'kv_handle_access_denied' });
|
||||
await expect(guestGet(handle, 'a')).rejects.toMatchObject({
|
||||
legacyCode: 'kv_handle_access_denied',
|
||||
});
|
||||
});
|
||||
|
||||
it('works for the owner through their own handle', async () => {
|
||||
const prefix = region('watch-owner');
|
||||
await ownerSet(`${prefix}a`, 1);
|
||||
const { handle } = await share(prefix, 'watch');
|
||||
await expect(
|
||||
as(owner.actor, (d) => d.list({ as: 'keys', ...via(handle) })),
|
||||
).resolves.toEqual(['a']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a handle whose owner is suspended', () => {
|
||||
const suspendable = async () =>
|
||||
userFor(
|
||||
await createTestUser(env.server, {
|
||||
username: `kv-susp-${uuidv4().slice(0, 8)}`,
|
||||
password: 'pw-test-1234',
|
||||
}),
|
||||
);
|
||||
|
||||
it('cannot be subscribed to, answered as not found', async () => {
|
||||
const suspendee = await suspendable();
|
||||
const { handle } = await share(region('susp-sub'), 'read', {
|
||||
actor: suspendee.actor,
|
||||
});
|
||||
await env.server.stores.user.update(suspendee.id, { suspended: 1 });
|
||||
|
||||
await expect(
|
||||
events().subscribe(guest.actor, 'kv-susp-socket', {
|
||||
subject: `kv:${handle}:*`,
|
||||
}),
|
||||
).rejects.toMatchObject({ legacyCode: 'subject_does_not_exist' });
|
||||
});
|
||||
|
||||
it('stops delivering to a subscription made before the suspension', async () => {
|
||||
const suspendee = await suspendable();
|
||||
const prefix = region('susp-deliver');
|
||||
const { handle } = await share(prefix, 'read', {
|
||||
actor: suspendee.actor,
|
||||
});
|
||||
const socket = 'kv-susp-deliver-socket';
|
||||
const { sub } = await events().subscribe(guest.actor, socket, {
|
||||
subject: `kv:${handle}:*`,
|
||||
});
|
||||
const writes = (key: string) =>
|
||||
as(suspendee.actor, (d) =>
|
||||
d.set({ key: `${prefix}${key}`, value: 1 }),
|
||||
);
|
||||
const reached = () => delivered.some((one) => one.subId === sub.subId);
|
||||
|
||||
delivered.length = 0;
|
||||
await writes('a');
|
||||
await vi.waitFor(() => expect(reached()).toBe(true), {
|
||||
timeout: EVENTS_COALESCE_WINDOW_MS * 12,
|
||||
interval: 25,
|
||||
});
|
||||
|
||||
// The cached row once it refreshes after the suspension.
|
||||
const live = await handles().getLive(handle);
|
||||
const getLive = vi
|
||||
.spyOn(handles(), 'getLive')
|
||||
.mockResolvedValue({ ...live!, ownerSuspended: true });
|
||||
try {
|
||||
delivered.length = 0;
|
||||
await writes('b');
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, EVENTS_COALESCE_WINDOW_MS * 3),
|
||||
);
|
||||
expect(reached()).toBe(false);
|
||||
} finally {
|
||||
getLive.mockRestore();
|
||||
await events().reapSocket(guest.id, socket);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -83,6 +83,8 @@ interface KvOptConfig {
|
||||
*/
|
||||
type KvCallArgs = {
|
||||
optConfig?: KvOptConfig;
|
||||
/** `list` only: what the listing returns. */
|
||||
as?: unknown;
|
||||
expireAt?: unknown;
|
||||
ttl?: unknown;
|
||||
};
|
||||
@@ -94,8 +96,7 @@ type KvCallArgs = {
|
||||
type DriverKvOpts = KVOpts & { handle?: string; handlePrefix?: string };
|
||||
|
||||
type KvListResult =
|
||||
| unknown[]
|
||||
| { items: unknown[]; cursor?: string; total?: number };
|
||||
unknown[] | { items: unknown[]; cursor?: string; total?: number };
|
||||
|
||||
/**
|
||||
* A listing of another namespace, cross-app or through a handle: what its
|
||||
@@ -404,7 +405,7 @@ export class KVStoreDriver extends PuterDriver {
|
||||
if (this.config.kv?.shareHandles !== true) throw kvHandlesDisabled();
|
||||
|
||||
const optConfig = args.optConfig ?? {};
|
||||
const level = kvShareLevelForMethod(method);
|
||||
const level = kvShareLevelForMethod(method, args);
|
||||
if (optConfig.appUuid || optConfig.disableSharing)
|
||||
throw handleOptionInvalid(
|
||||
'kv: `handle` cannot be combined with `appUuid` or `disableSharing`',
|
||||
@@ -553,8 +554,7 @@ export class KVStoreDriver extends PuterDriver {
|
||||
const decoded = decodeCursor(cursor);
|
||||
const reverse = decoded?.reverse === true;
|
||||
const position = (reverse ? decoded?.key : decoded) as
|
||||
| { namespace?: unknown; key?: unknown }
|
||||
| undefined;
|
||||
{ namespace?: unknown; key?: unknown } | undefined;
|
||||
if (position?.namespace !== listing.namespace) throw outsideRegion();
|
||||
const key = this.#relativeKey(position.key, listing.prefix);
|
||||
return sealCursor(
|
||||
|
||||
@@ -0,0 +1,530 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { makeActor, type Actor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { findNotificationType } from '../../services/notification/notificationTypes.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
import { peekRateLimit } from '../../core/http/middleware/rateLimit.js';
|
||||
import type { NotificationDriver } from './NotificationDriver.js';
|
||||
|
||||
/**
|
||||
* Charges to a held key wait here until the test lets them go, so two requests
|
||||
* can both be read before either is spent. Every charge is recorded by key.
|
||||
*/
|
||||
const charges = vi.hoisted(() => ({
|
||||
held: new Set<string>(),
|
||||
waiting: [] as Array<() => void>,
|
||||
made: [] as string[],
|
||||
}));
|
||||
|
||||
vi.mock('../../core/http/middleware/rateLimit.js', async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<
|
||||
typeof import('../../core/http/middleware/rateLimit.js')
|
||||
>();
|
||||
return {
|
||||
...actual,
|
||||
checkRateLimit: async (key: string, ...rest: unknown[]) => {
|
||||
if (charges.held.has(key)) {
|
||||
await new Promise<void>((go) => charges.waiting.push(go));
|
||||
}
|
||||
charges.made.push(key);
|
||||
return (
|
||||
actual.checkRateLimit as (...a: unknown[]) => Promise<boolean>
|
||||
)(key, ...rest);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const until = async (ready: () => boolean): Promise<void> => {
|
||||
for (let i = 0; i < 400 && !ready(); i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
if (!ready()) throw new Error('timed out');
|
||||
};
|
||||
|
||||
const makeUser = async (server: PuterServer): Promise<Actor> => {
|
||||
const username = `am-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const created = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
const user = (await server.stores.user.getById(created.id))!;
|
||||
return makeActor({
|
||||
user: {
|
||||
id: user.id,
|
||||
uuid: user.uuid,
|
||||
username: user.username,
|
||||
email: user.email ?? null,
|
||||
email_confirmed: true,
|
||||
} as Actor['user'],
|
||||
});
|
||||
};
|
||||
|
||||
const makeApp = async (
|
||||
server: PuterServer,
|
||||
ownerUserId: number,
|
||||
): Promise<{ id: number; uid: string }> => {
|
||||
const name = `am-app-${Math.random().toString(36).slice(2, 10)}`;
|
||||
return (await server.stores.app.create(
|
||||
{
|
||||
name,
|
||||
title: 'App message test',
|
||||
index_url: `https://${name}.test/`,
|
||||
},
|
||||
{ ownerUserId },
|
||||
)) as { id: number; uid: string };
|
||||
};
|
||||
|
||||
const asApp = (holder: Actor, app: { id: number; uid: string }): Actor =>
|
||||
makeActor({ user: holder.user, app });
|
||||
|
||||
/** A worker acts as its user, with its app as the app. */
|
||||
const asWorker = (holder: Actor, app: { id: number; uid: string }): Actor =>
|
||||
makeActor({
|
||||
user: holder.user,
|
||||
app,
|
||||
session: { uid: `ws-${uuidv4()}`, kind: 'worker' },
|
||||
});
|
||||
|
||||
const asIssuedToken = (holder: Actor, issuer: Actor): Actor =>
|
||||
makeActor({
|
||||
user: holder.user,
|
||||
accessToken: { uid: `tok-${uuidv4()}`, issuer, authorized: null },
|
||||
});
|
||||
|
||||
const withActor = <T>(actor: Actor, fn: () => Promise<T>): Promise<T> =>
|
||||
runWithContext({ actor }, fn);
|
||||
|
||||
const message = (fields: Record<string, unknown> = {}) => ({
|
||||
object: { value: { type: 'app.message', title: 'Hello', ...fields } },
|
||||
});
|
||||
|
||||
/** Every payload the bus carried under `key` while `fn` ran. */
|
||||
const captureBus = async (
|
||||
server: PuterServer,
|
||||
key: string,
|
||||
fn: () => Promise<unknown>,
|
||||
): Promise<Array<Record<string, unknown>>> => {
|
||||
const seen: Array<Record<string, unknown>> = [];
|
||||
const handler = (_k: string, data: unknown) =>
|
||||
seen.push(data as Record<string, unknown>);
|
||||
server.clients.event.on(key, handler);
|
||||
try {
|
||||
await fn();
|
||||
// Listeners may run after `emit` returns.
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
} finally {
|
||||
server.clients.event.off?.(key, handler);
|
||||
}
|
||||
return seen;
|
||||
};
|
||||
|
||||
const bootServer = (config: Partial<IConfig>) => {
|
||||
const state = {
|
||||
server: null as unknown as PuterServer,
|
||||
driver: null as unknown as NotificationDriver,
|
||||
};
|
||||
beforeAll(async () => {
|
||||
state.server = await setupTestServer(config as IConfig);
|
||||
state.driver = state.server.drivers
|
||||
.notifications as unknown as NotificationDriver;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await state.server?.shutdown();
|
||||
});
|
||||
return state;
|
||||
};
|
||||
|
||||
describe('app.message with live delivery over the socket wire', () => {
|
||||
const state = bootServer({
|
||||
notificationAppMessageLimits: { perMinute: 3 },
|
||||
});
|
||||
|
||||
it('stores the message for the acting user, under the app', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
|
||||
let created: Record<string, unknown> | null = null;
|
||||
const pushed = await captureBus(
|
||||
state.server,
|
||||
'outer.gui.notif.message',
|
||||
async () => {
|
||||
created = (await withActor(asApp(holder, app), () =>
|
||||
state.driver.create(
|
||||
message({ text: 'Body', channel: 'replies' }),
|
||||
),
|
||||
)) as Record<string, unknown>;
|
||||
},
|
||||
);
|
||||
|
||||
const uid = created!.uid as string;
|
||||
expect(created!.value).toEqual({
|
||||
type: 'app.message',
|
||||
title: 'Hello',
|
||||
text: 'Body',
|
||||
channel: 'replies',
|
||||
priority: 'normal',
|
||||
ttl: 3600,
|
||||
app: { uid: app.uid, name: 'App message test' },
|
||||
});
|
||||
const row = await state.server.stores.notification.getByUid(uid, {
|
||||
userId: holder.user.id,
|
||||
});
|
||||
expect(row).toMatchObject({
|
||||
user_id: holder.user.id,
|
||||
app_uid: app.uid,
|
||||
audience: 'app-user',
|
||||
type: 'app.message',
|
||||
});
|
||||
// Not silent: it went out on the wire the desktop listens to.
|
||||
expect(pushed).toEqual([
|
||||
{
|
||||
user_id_list: [holder.user.id],
|
||||
response: { uid, notification: created!.value },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("lets the app's worker and the tokens it issued notify their user", async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
|
||||
for (const actor of [
|
||||
asWorker(holder, app),
|
||||
asIssuedToken(holder, asApp(holder, app)),
|
||||
]) {
|
||||
const created = (await withActor(actor, () =>
|
||||
state.driver.create(message()),
|
||||
)) as Record<string, unknown>;
|
||||
const row = await state.server.stores.notification.getByUid(
|
||||
created.uid as string,
|
||||
{ userId: holder.user.id },
|
||||
);
|
||||
expect(row).toMatchObject({
|
||||
user_id: holder.user.id,
|
||||
app_uid: app.uid,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses an app any other type', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
for (const type of ['share.received', 'app.events.suspended']) {
|
||||
await expect(
|
||||
withActor(asApp(holder, app), () =>
|
||||
state.driver.create({ object: { value: { type } } }),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
code: 'notification_type_not_allowed',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses a recipient or app named by the caller', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const other = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
const otherApp = await makeApp(state.server, holder.user.id!);
|
||||
|
||||
for (const extra of [
|
||||
{ userId: other.user.id },
|
||||
{ recipient: other.user.uuid },
|
||||
{ appUid: otherApp.uid },
|
||||
]) {
|
||||
await expect(
|
||||
withActor(asApp(holder, app), () =>
|
||||
state.driver.create({
|
||||
object: { ...message().object, ...extra },
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
}
|
||||
const theirs = await state.server.stores.notification.listByUserId(
|
||||
other.user.id!,
|
||||
{},
|
||||
);
|
||||
expect(theirs).toEqual([]);
|
||||
});
|
||||
|
||||
it('refuses an app message that fails validation, writing nothing', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
await expect(
|
||||
withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message({ icon: 'http://x.test/a.png' })),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400, fields: { field: 'icon' } });
|
||||
await expect(
|
||||
withActor(asApp(holder, app), () =>
|
||||
state.driver.create({
|
||||
object: { value: { type: 'app.message' } },
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
fields: { field: 'title' },
|
||||
});
|
||||
expect(
|
||||
await state.server.stores.notification.listByUserId(
|
||||
holder.user.id!,
|
||||
{},
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the holder creating other types silently', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const pushed = await captureBus(
|
||||
state.server,
|
||||
'outer.gui.notif.message',
|
||||
() =>
|
||||
withActor(holder, () =>
|
||||
state.driver.create({
|
||||
object: {
|
||||
value: { type: 'share.received', title: 'hi' },
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(pushed).toEqual([]);
|
||||
const rows = await state.server.stores.notification.listByUserId(
|
||||
holder.user.id!,
|
||||
{},
|
||||
);
|
||||
expect(rows.map((r: Record<string, unknown>) => r.type)).toEqual([
|
||||
'share.received',
|
||||
]);
|
||||
});
|
||||
|
||||
it('refuses an app message from the holder, with or without an app', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
for (const object of [
|
||||
message().object,
|
||||
{ ...message().object, appUid: app.uid },
|
||||
]) {
|
||||
await expect(
|
||||
withActor(holder, () => state.driver.create({ object })),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
code: 'notification_type_not_allowed',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('limits each app per user per minute', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const otherUser = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
const otherApp = await makeApp(state.server, holder.user.id!);
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
);
|
||||
}
|
||||
// The app's worker spends the same budget as the app.
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await expect(
|
||||
withActor(asWorker(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 429,
|
||||
legacyCode: 'too_many_requests',
|
||||
});
|
||||
}
|
||||
// Refused calls left the day window at 3, not 5.
|
||||
expect(
|
||||
await peekRateLimit(
|
||||
`notif:app-message:day:${holder.user.id}:${app.uid}`,
|
||||
4,
|
||||
24 * 60 * 60_000,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
await expect(
|
||||
withActor(asApp(holder, otherApp), () =>
|
||||
state.driver.create(message()),
|
||||
),
|
||||
).resolves.toMatchObject({ uid: expect.any(String) });
|
||||
await expect(
|
||||
withActor(asApp(otherUser, app), () =>
|
||||
state.driver.create(message()),
|
||||
),
|
||||
).resolves.toMatchObject({ uid: expect.any(String) });
|
||||
});
|
||||
|
||||
it('spends the day window only on a request the minute window admits', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
const minuteKey = `notif:app-message:minute:${holder.user.id}:${app.uid}`;
|
||||
const dayKey = `notif:app-message:day:${holder.user.id}:${app.uid}`;
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
);
|
||||
}
|
||||
|
||||
charges.held.add(minuteKey).add(dayKey);
|
||||
charges.made.length = 0;
|
||||
let settled = false;
|
||||
const outcome = Promise.allSettled([
|
||||
withActor(asApp(holder, app), () => state.driver.create(message())),
|
||||
withActor(asApp(holder, app), () => state.driver.create(message())),
|
||||
]).finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
try {
|
||||
// Both requests read the windows as open before either spends.
|
||||
await until(() => charges.waiting.length === 2);
|
||||
while (!settled) {
|
||||
await until(() => settled || charges.waiting.length > 0);
|
||||
charges.waiting.shift()?.();
|
||||
}
|
||||
} finally {
|
||||
charges.held.clear();
|
||||
}
|
||||
|
||||
const results = await outcome;
|
||||
expect(results.map((r) => r.status).sort()).toEqual([
|
||||
'fulfilled',
|
||||
'rejected',
|
||||
]);
|
||||
const refused = results.find((r) => r.status === 'rejected') as
|
||||
PromiseRejectedResult | undefined;
|
||||
expect(refused?.reason).toMatchObject({ statusCode: 429 });
|
||||
// The refused request left the day window alone.
|
||||
expect(charges.made.filter((k) => k === dayKey)).toEqual([dayKey]);
|
||||
expect(await peekRateLimit(dayKey, 4, 24 * 60 * 60_000)).toBe(true);
|
||||
});
|
||||
|
||||
it("reads back the app's own messages and no other app's", async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
const otherApp = await makeApp(state.server, holder.user.id!);
|
||||
const created = (await withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
)) as Record<string, unknown>;
|
||||
await withActor(holder, () =>
|
||||
state.driver.create({
|
||||
object: { value: { type: 'share.received', title: 'acct' } },
|
||||
}),
|
||||
);
|
||||
|
||||
const read = (await withActor(asApp(holder, app), () =>
|
||||
state.driver.read({ uid: created.uid }),
|
||||
)) as Record<string, unknown>;
|
||||
expect(read.uid).toBe(created.uid);
|
||||
|
||||
for (const subject of [undefined, 'notif:app-user']) {
|
||||
const listed = (await withActor(asApp(holder, app), () =>
|
||||
state.driver.select({ subject }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
expect(listed.map((n) => n.uid)).toEqual([created.uid]);
|
||||
}
|
||||
|
||||
await expect(
|
||||
withActor(asApp(holder, otherApp), () =>
|
||||
state.driver.read({ uid: created.uid }),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
expect(
|
||||
await withActor(asApp(holder, otherApp), () =>
|
||||
state.driver.select({}),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('app.message with the events fold-in on', () => {
|
||||
const state = bootServer({
|
||||
events: { enabled: true, notificationsFoldIn: true },
|
||||
notificationAppMessageLimits: { perMinute: 3, perDay: 2 },
|
||||
} as Partial<IConfig>);
|
||||
|
||||
it("emits notif.created on the app's app-user subject", async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
let uid = '';
|
||||
const created = await captureBus(
|
||||
state.server,
|
||||
'notif.created',
|
||||
async () => {
|
||||
const result = (await withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
)) as Record<string, unknown>;
|
||||
uid = result.uid as string;
|
||||
},
|
||||
);
|
||||
|
||||
expect(created).toEqual([
|
||||
expect.objectContaining({
|
||||
userId: holder.user.id,
|
||||
uid,
|
||||
type: 'app.message',
|
||||
audience: 'app-user',
|
||||
appUid: app.uid,
|
||||
}),
|
||||
]);
|
||||
expect(
|
||||
findNotificationType('app.message')!.subject({
|
||||
userUuid: holder.user.uuid!,
|
||||
appUid: app.uid,
|
||||
}),
|
||||
).toBe(`notif:${app.uid}:app-user`);
|
||||
});
|
||||
|
||||
it('limits each app per user per day, without spending the minute', async () => {
|
||||
const holder = await makeUser(state.server);
|
||||
const app = await makeApp(state.server, holder.user.id!);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
);
|
||||
}
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await expect(
|
||||
withActor(asApp(holder, app), () =>
|
||||
state.driver.create(message()),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 429 });
|
||||
}
|
||||
// Refused calls left the minute window at 2, under its limit of 3.
|
||||
expect(
|
||||
await peekRateLimit(
|
||||
`notif:app-message:minute:${holder.user.id}:${app.uid}`,
|
||||
3,
|
||||
60_000,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -222,7 +222,7 @@ describe('NotificationDriver.create', () => {
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('rejects an app-actor with 403', async () => {
|
||||
it('rejects an app creating anything but an app message with 403', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const appActor = makeActor({
|
||||
user: actor.user,
|
||||
@@ -237,7 +237,7 @@ describe('NotificationDriver.create', () => {
|
||||
).rejects.toMatchObject({ statusCode: 403 });
|
||||
});
|
||||
|
||||
it('rejects a token an app issued with 403', async () => {
|
||||
it('rejects a token an app issued creating another type with 403', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const issued = asIssuedToken(
|
||||
actor,
|
||||
@@ -689,6 +689,16 @@ describe('NotificationDriver — read per token type', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('returns the app a row is about, and null for a row naming none', async () => {
|
||||
const box = await mailbox();
|
||||
expect(await readUid(box.holder.actor, box.rows.appUser)).toMatchObject(
|
||||
{ app_uid: box.ownApp.uid },
|
||||
);
|
||||
expect(await readUid(box.holder.actor, box.rows.account)).toMatchObject(
|
||||
{ app_uid: null },
|
||||
);
|
||||
});
|
||||
|
||||
it('an app reads only what it is the subject of', async () => {
|
||||
const box = await mailbox();
|
||||
expect(await readUid(box.ownAppActor, box.rows.appUser)).toMatchObject({
|
||||
|
||||
@@ -23,11 +23,21 @@ import {
|
||||
type UserActor,
|
||||
} from '../../core/actor.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import {
|
||||
checkRateLimit,
|
||||
peekRateLimit,
|
||||
} from '../../core/http/middleware/rateLimit.js';
|
||||
import { resolveNotifFetch } from '../../services/events/notifFetch.js';
|
||||
import {
|
||||
DEFAULT_FREE_SUBSCRIPTION,
|
||||
DEFAULT_TEMP_SUBSCRIPTION,
|
||||
} from '../../services/metering/consts.js';
|
||||
import {
|
||||
APP_MESSAGE_RATE_LIMITS,
|
||||
APP_MESSAGE_TYPE,
|
||||
appMessageSender,
|
||||
parseAppMessage,
|
||||
} from '../../services/notification/appMessage.js';
|
||||
import {
|
||||
APP_READABLE_AUDIENCES,
|
||||
canViewNotification,
|
||||
@@ -46,6 +56,8 @@ import type { DriverRateLimitConfig } from '../meta.js';
|
||||
import { CRUD_CONCURRENT } from '../util/crudLimits.js';
|
||||
|
||||
const MAX_SELECT_LIMIT = 200;
|
||||
const MINUTE_MS = 60_000;
|
||||
const DAY_MS = 24 * 60 * MINUTE_MS;
|
||||
|
||||
/** The mailbox slice a call named, in the terms the query takes it. */
|
||||
interface MailboxScope {
|
||||
@@ -54,6 +66,12 @@ interface MailboxScope {
|
||||
appUid: string | null | undefined;
|
||||
}
|
||||
|
||||
const typeNotAllowed = (message: string): HttpError =>
|
||||
new HttpError(403, message, {
|
||||
legacyCode: 'forbidden',
|
||||
code: 'notification_type_not_allowed',
|
||||
});
|
||||
|
||||
/** Whether a row belongs to the slice, as the same SQL scope selected it. */
|
||||
const inScope = (
|
||||
row: Record<string, unknown>,
|
||||
@@ -83,7 +101,9 @@ const inScope = (
|
||||
* owns the app. `account` rows never reach an actor holding an app, and no
|
||||
* grant changes that. A slice an actor may not see comes back empty rather
|
||||
* than refused — the mailbox is not an oracle.
|
||||
* - `create` stays holder-only.
|
||||
* - The holder creates any registered type but `app.message`, silently. An actor
|
||||
* holding an app creates `app.message` only, for its own user, and it is
|
||||
* delivered live.
|
||||
*
|
||||
* `read` and `select` also take an optional `subject` naming one slice
|
||||
* (`notif:<appUid>:<audience>`); the two-segment form expands from the actor's
|
||||
@@ -126,7 +146,7 @@ export class NotificationDriver extends PuterDriver {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
const actor = this.#requireUserActor();
|
||||
const actor = this.#requireActor();
|
||||
|
||||
const value = (object.value ?? {}) as Record<string, unknown>;
|
||||
const type = value.type;
|
||||
@@ -146,6 +166,18 @@ export class NotificationDriver extends PuterDriver {
|
||||
);
|
||||
}
|
||||
|
||||
if (actor.effectiveApp) {
|
||||
return this.#createAppMessage(
|
||||
actor,
|
||||
actor.effectiveApp.uid,
|
||||
object,
|
||||
value,
|
||||
);
|
||||
}
|
||||
if (type === APP_MESSAGE_TYPE) {
|
||||
throw typeNotAllowed(`Only apps can create ${APP_MESSAGE_TYPE}`);
|
||||
}
|
||||
|
||||
const appUid = (object.appUid ?? null) as string | null;
|
||||
// The registry rejects with a plain Error — unregistered, or a type
|
||||
// wrong for the audience it names. Both are caller input, so neither
|
||||
@@ -166,23 +198,11 @@ export class NotificationDriver extends PuterDriver {
|
||||
// Silent: a driver `create` has never pushed to the live socket, only
|
||||
// ever written the row — moving through the registry does not change
|
||||
// that.
|
||||
const uid = await this.services.notification.notify(
|
||||
[actor.user.id],
|
||||
value,
|
||||
{ type: registeredType, appUid, silent: true },
|
||||
);
|
||||
if (!uid) {
|
||||
throw new HttpError(500, 'Notification could not be created', {
|
||||
legacyCode: 'internal_error',
|
||||
});
|
||||
}
|
||||
// `notify` hands back only the uid; the primary has the rest.
|
||||
return this.#toClient(
|
||||
await this.stores.notification.getByUid(uid, {
|
||||
userId: actor.user.id,
|
||||
primary: true,
|
||||
}),
|
||||
);
|
||||
return this.#write(actor, value, {
|
||||
type: registeredType,
|
||||
appUid,
|
||||
silent: true,
|
||||
});
|
||||
}
|
||||
|
||||
async read(args: Record<string, unknown>): Promise<unknown> {
|
||||
@@ -280,6 +300,107 @@ export class NotificationDriver extends PuterDriver {
|
||||
return { success: !!ok };
|
||||
}
|
||||
|
||||
/** An app's message to its own user; the actor names recipient and app. */
|
||||
async #createAppMessage(
|
||||
actor: UserActor,
|
||||
appUid: string,
|
||||
object: Record<string, unknown>,
|
||||
value: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
const { type, ...fields } = value;
|
||||
if (type !== APP_MESSAGE_TYPE) {
|
||||
throw typeNotAllowed(`Apps can create only ${APP_MESSAGE_TYPE}`);
|
||||
}
|
||||
for (const key of Object.keys(object)) {
|
||||
if (key !== 'value') {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`\`${key}\` is set by the server for app notifications`,
|
||||
{ legacyCode: 'bad_request', fields: { field: key } },
|
||||
);
|
||||
}
|
||||
}
|
||||
const message = parseAppMessage(fields);
|
||||
|
||||
if (!(await this.#withinAppMessageLimits(actor.user.id, appUid))) {
|
||||
throw new HttpError(429, 'Too many notifications from this app.', {
|
||||
legacyCode: 'too_many_requests',
|
||||
});
|
||||
}
|
||||
|
||||
message.app = appMessageSender(
|
||||
appUid,
|
||||
await this.stores.app.getByUid(appUid),
|
||||
);
|
||||
|
||||
return this.#write(actor, message, {
|
||||
type: APP_MESSAGE_TYPE,
|
||||
appUid,
|
||||
silent: false,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Both windows are read before either is spent, so a refusal costs none.
|
||||
* The minute window is spent first: it is the one most likely to run out
|
||||
* between the read and the spend, and a refusal there leaves the day
|
||||
* alone.
|
||||
*/
|
||||
async #withinAppMessageLimits(
|
||||
userId: number,
|
||||
appUid: string,
|
||||
): Promise<boolean> {
|
||||
const configured = this.config.notificationAppMessageLimits ?? {};
|
||||
const all: Array<[string, number, number]> = [
|
||||
[
|
||||
`notif:app-message:minute:${userId}:${appUid}`,
|
||||
configured.perMinute ?? APP_MESSAGE_RATE_LIMITS.perMinute,
|
||||
MINUTE_MS,
|
||||
],
|
||||
[
|
||||
`notif:app-message:day:${userId}:${appUid}`,
|
||||
configured.perDay ?? APP_MESSAGE_RATE_LIMITS.perDay,
|
||||
DAY_MS,
|
||||
],
|
||||
];
|
||||
const windows = all.filter(([, limit]) => limit > 0);
|
||||
for (const [key, limit, windowMs] of windows) {
|
||||
if (!(await peekRateLimit(key, limit, windowMs))) return false;
|
||||
}
|
||||
for (const [key, limit, windowMs] of windows) {
|
||||
if (!(await checkRateLimit(key, limit, windowMs))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async #write(
|
||||
actor: UserActor,
|
||||
value: Record<string, unknown>,
|
||||
opts: {
|
||||
type: NotificationTypeName;
|
||||
appUid: string | null;
|
||||
silent: boolean;
|
||||
},
|
||||
): Promise<unknown> {
|
||||
const uid = await this.services.notification.notify(
|
||||
[actor.user.id],
|
||||
value,
|
||||
opts,
|
||||
);
|
||||
if (!uid) {
|
||||
throw new HttpError(500, 'Notification could not be created', {
|
||||
legacyCode: 'internal_error',
|
||||
});
|
||||
}
|
||||
// `notify` hands back only the uid; the primary has the rest.
|
||||
return this.#toClient(
|
||||
await this.stores.notification.getByUid(uid, {
|
||||
userId: actor.user.id,
|
||||
primary: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// -- Permissions -------------------------------------------------
|
||||
|
||||
#requireActor(): UserActor {
|
||||
@@ -294,16 +415,6 @@ export class NotificationDriver extends PuterDriver {
|
||||
return actor as UserActor;
|
||||
}
|
||||
|
||||
/** Writing a notification stays the holder's, per the token table. */
|
||||
#requireUserActor(): UserActor {
|
||||
const actor = this.#requireActor();
|
||||
if (actor.effectiveApp)
|
||||
throw new HttpError(403, 'App actors cannot create notifications', {
|
||||
legacyCode: 'forbidden',
|
||||
});
|
||||
return actor;
|
||||
}
|
||||
|
||||
/**
|
||||
* The slice a call named, or `null` for the holder's whole mailbox. An
|
||||
* actor holding an app always has a slice — its own — so `null` only ever
|
||||
@@ -395,6 +506,7 @@ export class NotificationDriver extends PuterDriver {
|
||||
return {
|
||||
uid: row.uid,
|
||||
value: row.value,
|
||||
app_uid: row.app_uid ?? null,
|
||||
shown: row.shown ?? null,
|
||||
acknowledged: row.acknowledged ?? null,
|
||||
created_at: row.created_at ?? null,
|
||||
|
||||
@@ -573,6 +573,24 @@ describe('SubdomainDriver.update field handling', () => {
|
||||
expect(cleared.associated_app).toBeNull();
|
||||
});
|
||||
|
||||
it('disconnects the directory when root_dir is null', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const name = uniqueName('unlink');
|
||||
await createSubdomain(actor, name);
|
||||
|
||||
// The shape `puter.hosting.update(name, null)` sends.
|
||||
const updated = (await withActor(actor, () =>
|
||||
driver.update({
|
||||
id: { subdomain: name },
|
||||
object: { root_dir: null },
|
||||
}),
|
||||
)) as Record<string, unknown>;
|
||||
|
||||
expect(updated.root_dir).toBeNull();
|
||||
const row = await server.stores.subdomain.getBySubdomain(name);
|
||||
expect(row?.root_dir_id ?? null).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects repointing root_dir at a path that does not exist', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const created = await createSubdomain(actor, uniqueName('badroot'));
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { makeActor } from '../../core/actor.js';
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { SubdomainDriver } from './SubdomainDriver.js';
|
||||
|
||||
// `puter.perms.request('subdomains', { access })` grants an app
|
||||
// `subdomains-of-user:<uuid>:read|write`, widening its scope from the rows it
|
||||
// created to all of its user's rows, and never past them.
|
||||
|
||||
let server: PuterServer;
|
||||
let driver: SubdomainDriver;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
driver = server.drivers.subdomains as unknown as SubdomainDriver;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
const rand = () => Math.random().toString(36).slice(2, 10);
|
||||
|
||||
const makeUser = async () => {
|
||||
const username = `sdg-${rand()}`;
|
||||
const created = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
const user = (await server.stores.user.getById(created.id))!;
|
||||
return makeActor({ user: { ...user, email_confirmed: true } as never });
|
||||
};
|
||||
|
||||
const makeApp = async (owner: Actor) =>
|
||||
(await server.stores.app.create(
|
||||
{
|
||||
name: `sdg-app-${uuidv4()}`,
|
||||
title: 'Site manager',
|
||||
index_url: `https://sdg-${uuidv4()}.example.test/`,
|
||||
},
|
||||
{ ownerUserId: owner.user!.id },
|
||||
)) as { uid: string; id: number };
|
||||
|
||||
const asApp = (user: Actor, app: { uid: string; id: number }) =>
|
||||
makeActor({ ...user, app: { uid: app.uid, id: app.id } });
|
||||
|
||||
const seedRow = (
|
||||
user: Actor,
|
||||
appOwner: number | null,
|
||||
name = `sdg-${rand()}`,
|
||||
) =>
|
||||
server.stores.subdomain.create({
|
||||
userId: user.user!.id,
|
||||
subdomain: name,
|
||||
rootDirId: null,
|
||||
associatedAppId: null,
|
||||
appOwner,
|
||||
});
|
||||
|
||||
const grant = (user: Actor, app: { uid: string }, access: 'read' | 'write') =>
|
||||
server.services.permission.grantUserAppPermission(
|
||||
user,
|
||||
app.uid,
|
||||
`subdomains-of-user:${user.user!.uuid}:${access}`,
|
||||
);
|
||||
|
||||
const as = <T>(actor: Actor, fn: () => Promise<T>) =>
|
||||
runWithContext({ actor }, fn);
|
||||
|
||||
const listed = async (actor: Actor) =>
|
||||
(
|
||||
(await as(actor, () => driver.select({}))) as Array<{
|
||||
subdomain: string;
|
||||
}>
|
||||
).map((row) => row.subdomain);
|
||||
|
||||
describe('SubdomainDriver with a subdomains grant', () => {
|
||||
it('leaves an app without the grant scoped to the rows it created', async () => {
|
||||
const user = await makeUser();
|
||||
const app = await makeApp(user);
|
||||
const own = await seedRow(user, app.id);
|
||||
const loose = await seedRow(user, null);
|
||||
const appActor = asApp(user, app);
|
||||
|
||||
expect(await listed(appActor)).toEqual([own.subdomain]);
|
||||
const page = (await as(appActor, () =>
|
||||
driver.select({ includeTotal: true }),
|
||||
)) as { items: unknown[]; total: number };
|
||||
expect(page.total).toBe(1);
|
||||
for (const call of [
|
||||
() => driver.read({ uid: loose.uuid }),
|
||||
() => driver.update({ uid: loose.uuid, object: {} }),
|
||||
() => driver.delete({ uid: loose.uuid }),
|
||||
]) {
|
||||
await expect(as(appActor, call)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('read lets the app list and read all of its user rows, but not change them', async () => {
|
||||
const user = await makeUser();
|
||||
const app = await makeApp(user);
|
||||
const otherApp = await makeApp(user);
|
||||
const own = await seedRow(user, app.id);
|
||||
const loose = await seedRow(user, null);
|
||||
const byOtherApp = await seedRow(user, otherApp.id);
|
||||
await grant(user, app, 'read');
|
||||
const appActor = asApp(user, app);
|
||||
|
||||
expect((await listed(appActor)).sort()).toEqual(
|
||||
[own.subdomain, loose.subdomain, byOtherApp.subdomain].sort(),
|
||||
);
|
||||
const page = (await as(appActor, () =>
|
||||
driver.select({ includeTotal: true }),
|
||||
)) as { total: number };
|
||||
expect(page.total).toBe(3);
|
||||
const read = (await as(appActor, () =>
|
||||
driver.read({ id: { subdomain: loose.subdomain } }),
|
||||
)) as { uid: string };
|
||||
expect(read.uid).toBe(loose.uuid);
|
||||
|
||||
for (const call of [
|
||||
() => driver.update({ uid: loose.uuid, object: {} }),
|
||||
() => driver.delete({ uid: byOtherApp.uuid }),
|
||||
]) {
|
||||
await expect(as(appActor, call)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('write also lets the app change and delete them', async () => {
|
||||
const user = await makeUser();
|
||||
const app = await makeApp(user);
|
||||
const otherApp = await makeApp(user);
|
||||
const loose = await seedRow(user, null);
|
||||
const byOtherApp = await seedRow(user, otherApp.id);
|
||||
await grant(user, app, 'write');
|
||||
const appActor = asApp(user, app);
|
||||
|
||||
expect((await listed(appActor)).sort()).toEqual(
|
||||
[loose.subdomain, byOtherApp.subdomain].sort(),
|
||||
);
|
||||
const updated = (await as(appActor, () =>
|
||||
driver.update({ uid: loose.uuid, object: {} }),
|
||||
)) as { uid: string };
|
||||
expect(updated.uid).toBe(loose.uuid);
|
||||
await as(appActor, () => driver.delete({ uid: byOtherApp.uuid }));
|
||||
expect(
|
||||
await server.stores.subdomain.getByUuid(String(byOtherApp.uuid), {
|
||||
primary: true,
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('never reaches another user rows, or rows of a user who granted a different app', async () => {
|
||||
const user = await makeUser();
|
||||
const stranger = await makeUser();
|
||||
const app = await makeApp(user);
|
||||
const otherApp = await makeApp(user);
|
||||
const strangers = await seedRow(stranger, app.id);
|
||||
const mine = await seedRow(user, null);
|
||||
await grant(user, otherApp, 'write');
|
||||
// The stranger's own grant names the stranger, not `user`.
|
||||
await grant(stranger, app, 'write');
|
||||
|
||||
const appActor = asApp(user, app);
|
||||
expect(await listed(appActor)).toEqual([]);
|
||||
for (const call of [
|
||||
() => driver.read({ uid: mine.uuid }),
|
||||
() => driver.read({ uid: strangers.uuid }),
|
||||
() => driver.delete({ uid: strangers.uuid }),
|
||||
]) {
|
||||
await expect(as(appActor, call)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -87,7 +87,8 @@ const RESERVED_SUBDOMAINS = new Set([
|
||||
*
|
||||
* - Owner (user_id) can read/write their own subdomains
|
||||
* - An app actor is further scoped to the rows it created (app_owner), for reads
|
||||
* as well as writes — never widened past its own user
|
||||
* as well as writes, unless the user granted it
|
||||
* `subdomains-of-user:<uuid>:read|write` — never widened past its own user
|
||||
* - `system:es:write-all-owners` grants blanket write
|
||||
* - `read-all-subdomains` grants cross-user reads, and is the only thing that
|
||||
* does
|
||||
@@ -284,8 +285,15 @@ export class SubdomainDriver extends PuterDriver {
|
||||
predicate !== 'user-can-edit' &&
|
||||
(await this.#hasPermission(actor, 'read-all-subdomains'));
|
||||
|
||||
// App actors only see subdomains they own; read-all bypasses scoping.
|
||||
const appOwner = widenToAll ? undefined : actor.effectiveApp?.id;
|
||||
// App actors only see subdomains they own unless granted the user's;
|
||||
// read-all bypasses scoping.
|
||||
let appOwner = widenToAll ? undefined : actor.effectiveApp?.id;
|
||||
if (
|
||||
appOwner !== undefined &&
|
||||
(await this.#appHoldsUserSubdomains(actor, 'read'))
|
||||
) {
|
||||
appOwner = undefined;
|
||||
}
|
||||
// Worker deployments live in the same table but aren't sites —
|
||||
// they're listed through the workers driver instead.
|
||||
const listOpts = {
|
||||
@@ -357,7 +365,10 @@ export class SubdomainDriver extends PuterDriver {
|
||||
|
||||
// Subdomain name is immutable — strip if provided
|
||||
const patch: Record<string, unknown> = {};
|
||||
if (object.root_dir !== undefined) {
|
||||
// A null root_dir disconnects the directory; the site stays reserved.
|
||||
if (object.root_dir === null) {
|
||||
patch.root_dir_id = null;
|
||||
} else if (object.root_dir !== undefined) {
|
||||
const rootDirPath = expandTildePath(
|
||||
String(object.root_dir),
|
||||
actor.user.username,
|
||||
@@ -441,6 +452,7 @@ export class SubdomainDriver extends PuterDriver {
|
||||
{
|
||||
subdomain: row.subdomain as string,
|
||||
uid: String(row.uuid),
|
||||
userId: row.user_id as number,
|
||||
},
|
||||
{},
|
||||
);
|
||||
@@ -533,6 +545,21 @@ export class SubdomainDriver extends PuterDriver {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the user granted this app actor all of their own subdomains
|
||||
* (`puter.perms.request('subdomains')`). Write implies read.
|
||||
*/
|
||||
async #appHoldsUserSubdomains(
|
||||
actor: Actor,
|
||||
access: 'read' | 'write',
|
||||
): Promise<boolean> {
|
||||
if (!actor.effectiveApp?.id || !actor.user?.uuid) return false;
|
||||
return this.#hasPermission(
|
||||
actor,
|
||||
`subdomains-of-user:${actor.user.uuid}:${access}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Both grants are nested under "the caller owns this row" on purpose.
|
||||
*
|
||||
@@ -545,7 +572,8 @@ export class SubdomainDriver extends PuterDriver {
|
||||
* of its branches; this is the same rule, written as nesting.
|
||||
*
|
||||
* The inner check is the predicate `select` applies in SQL: an app sees
|
||||
* what it created, not everything its user owns.
|
||||
* what it created, not everything its user owns, unless the user granted it
|
||||
* their subdomains.
|
||||
*/
|
||||
async #checkReadAccess(
|
||||
row: Record<string, unknown>,
|
||||
@@ -555,6 +583,7 @@ export class SubdomainDriver extends PuterDriver {
|
||||
const app = actor.effectiveApp;
|
||||
if (!app?.id) return;
|
||||
if (app.id === row.app_owner) return;
|
||||
if (await this.#appHoldsUserSubdomains(actor, 'read')) return;
|
||||
}
|
||||
// Cross-user read permission
|
||||
if (await this.#hasPermission(actor, 'read-all-subdomains')) return;
|
||||
@@ -603,6 +632,12 @@ export class SubdomainDriver extends PuterDriver {
|
||||
appOwnerId: row.app_owner,
|
||||
});
|
||||
if (owns) return;
|
||||
if (
|
||||
actor.user?.id === row.user_id &&
|
||||
(await this.#appHoldsUserSubdomains(actor, 'write'))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (await this.#hasPermission(actor, 'system:es:write-all-owners')) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -651,6 +651,122 @@ describe('AI cost factor', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('renews the hold until a slow charge lands', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
|
||||
const refresh = vi.spyOn(server.stores.creditHold, 'refresh');
|
||||
const check = vi
|
||||
.spyOn(metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(true);
|
||||
let landCharge = () => {};
|
||||
const charge = new Promise<void>((resolve) => {
|
||||
landCharge = resolve;
|
||||
});
|
||||
try {
|
||||
const result = await withStreamedAiCharge(
|
||||
scoped,
|
||||
actor,
|
||||
usageType,
|
||||
1000,
|
||||
async () => ({
|
||||
dataType: 'stream',
|
||||
content_type: 'audio/mpeg',
|
||||
stream: Readable.from([Buffer.from('ab')]),
|
||||
}),
|
||||
() => charge,
|
||||
);
|
||||
expect(await read(result.stream)).toBeUndefined();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(11 * 60 * 1000);
|
||||
expect(refresh).toHaveBeenCalledTimes(2);
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(1000);
|
||||
|
||||
landCharge();
|
||||
await vi.waitFor(async () =>
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(0),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(10 * 60 * 1000);
|
||||
expect(refresh).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
refresh.mockRestore();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('renews the hold while a slow upstream is still opening', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
|
||||
const refresh = vi.spyOn(server.stores.creditHold, 'refresh');
|
||||
const check = vi
|
||||
.spyOn(metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(true);
|
||||
let opened = () => {};
|
||||
const opening = new Promise<void>((resolve) => {
|
||||
opened = resolve;
|
||||
});
|
||||
try {
|
||||
const pending = withStreamedAiCharge(
|
||||
scoped,
|
||||
actor,
|
||||
usageType,
|
||||
1000,
|
||||
async () => {
|
||||
await opening;
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: 'audio/mpeg',
|
||||
stream: Readable.from([Buffer.from('ab')]),
|
||||
};
|
||||
},
|
||||
() => undefined,
|
||||
);
|
||||
await vi.waitFor(async () =>
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(
|
||||
1000,
|
||||
),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(9 * 60 * 1000);
|
||||
expect(refresh).toHaveBeenCalledTimes(1);
|
||||
|
||||
opened();
|
||||
expect(await read((await pending).stream)).toBeUndefined();
|
||||
await vi.waitFor(async () =>
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(0),
|
||||
);
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
refresh.mockRestore();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('releases the hold when recording the charge throws', async () => {
|
||||
const check = vi
|
||||
.spyOn(metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(true);
|
||||
try {
|
||||
const result = await withStreamedAiCharge(
|
||||
scoped,
|
||||
actor,
|
||||
usageType,
|
||||
1000,
|
||||
async () => ({
|
||||
dataType: 'stream',
|
||||
content_type: 'audio/mpeg',
|
||||
stream: Readable.from([Buffer.from('ab')]),
|
||||
}),
|
||||
() => {
|
||||
throw new Error('metering down');
|
||||
},
|
||||
);
|
||||
expect(await read(result.stream)).toBeUndefined();
|
||||
await vi.waitFor(async () =>
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(0),
|
||||
);
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('charges nothing when the upstream fails before its first byte', async () => {
|
||||
const { record, stream } = await relay(
|
||||
new Readable({
|
||||
|
||||
@@ -416,10 +416,14 @@ export async function withAiCreditHold<T>(
|
||||
}
|
||||
}
|
||||
|
||||
// Under the hold store's default TTL, so a renewal always lands first.
|
||||
const STREAM_HOLD_RENEW_MS = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Holds `amount` while `open` starts an upstream audio stream, then charges it
|
||||
* through `record` once the first byte arrives, or once the caller hangs up. An
|
||||
* upstream that fails or ends before its first byte is not charged.
|
||||
* upstream that fails or ends before its first byte is not charged. The hold is
|
||||
* renewed until the charge lands, however long a slow reader takes.
|
||||
*/
|
||||
export async function withStreamedAiCharge(
|
||||
metering: AiMeteringService,
|
||||
@@ -431,10 +435,20 @@ export async function withStreamedAiCharge(
|
||||
): Promise<DriverStreamResult> {
|
||||
const hold = await metering.reserveAiCredits(actor, usageType, amount);
|
||||
if (!hold) throw insufficientCreditsError();
|
||||
// Started before `open`, which can itself run as long as the hold's TTL.
|
||||
const renew = setInterval(async () => {
|
||||
try {
|
||||
await hold.extend?.();
|
||||
} catch {
|
||||
// The next tick retries.
|
||||
}
|
||||
}, STREAM_HOLD_RENEW_MS);
|
||||
renew.unref?.();
|
||||
let result: DriverStreamResult;
|
||||
try {
|
||||
result = await open();
|
||||
} catch (e) {
|
||||
clearInterval(renew);
|
||||
await hold.release();
|
||||
throw e;
|
||||
}
|
||||
@@ -445,11 +459,17 @@ export async function withStreamedAiCharge(
|
||||
settled = true;
|
||||
// The relay runs outside the request, so the hold waits on the charge
|
||||
// itself. A failed release leaves the hold to its TTL.
|
||||
const release = () => hold.release().catch(() => {});
|
||||
void Promise.resolve(charge ? record() : undefined).then(
|
||||
release,
|
||||
release,
|
||||
);
|
||||
const release = async () => {
|
||||
clearInterval(renew);
|
||||
try {
|
||||
await hold.release();
|
||||
} catch {
|
||||
// ignored
|
||||
}
|
||||
};
|
||||
void Promise.resolve()
|
||||
.then(() => (charge ? record() : undefined))
|
||||
.then(release, release);
|
||||
};
|
||||
const source = result.stream;
|
||||
let chunks: AsyncIterator<unknown> | undefined;
|
||||
|
||||
Loaded 100 of 446 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user