mirror of
https://github.com/HeyPuter/puter.git
synced 2026-11-13 18:34:51 -10:00
Compare commits
76
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c31f9177c | ||
|
|
5531f381f5 | ||
|
|
8a198b6f0a | ||
|
|
17dc879ff5 | ||
|
|
d3321b1d0b | ||
|
|
a52f32e0d2 | ||
|
|
086e3a0028 | ||
|
|
aabbafdaab | ||
|
|
97ab0ee12b | ||
|
|
358b680b69 | ||
|
|
e534a3d09a | ||
|
|
5fd07e9ed7 | ||
|
|
e138f1e9d8 | ||
|
|
7886907bdf | ||
|
|
0eec08df98 | ||
|
|
4b5d6e034c | ||
|
|
7d53cc7ed6 | ||
|
|
86f871c9a8 | ||
|
|
c038a6983f | ||
|
|
f044d8d071 | ||
|
|
aea6e662f6 | ||
|
|
31f6dc2d37 | ||
|
|
2b03bc53b1 | ||
|
|
3ea0010220 | ||
|
|
395e4bd54a | ||
|
|
b55cb715ca | ||
|
|
2e4d918900 | ||
|
|
8c799a433a | ||
|
|
5f84d5897b | ||
|
|
c0aff857a4 | ||
|
|
ff606abf3c | ||
|
|
8159998830 | ||
|
|
395f7f6adb | ||
|
|
1c5909331c | ||
|
|
5d541cd8e2 | ||
|
|
fb526bfb7e | ||
|
|
1abd25eb31 | ||
|
|
b5e4f20087 | ||
|
|
bdb88507df | ||
|
|
0694b76751 | ||
|
|
e7ee778985 | ||
|
|
c3ff8ca3b2 | ||
|
|
39d931306d | ||
|
|
70e823c5af | ||
|
|
1bc1953344 | ||
|
|
0f83bb9b56 | ||
|
|
1ae95d97e8 | ||
|
|
31f8936da2 | ||
|
|
aab559c9c1 | ||
|
|
7f1d46028a | ||
|
|
39d9482409 | ||
|
|
4a958b3ca1 | ||
|
|
4653ff6dae | ||
|
|
a5c7b427b8 | ||
|
|
cd3600bae9 | ||
|
|
2db55bd636 | ||
|
|
36e2d199c8 | ||
|
|
8616b00f7b | ||
|
|
873303a7c4 | ||
|
|
11107f3108 | ||
|
|
2c3b69011f | ||
|
|
eefc1f2f2e | ||
|
|
7e093d2ba7 | ||
|
|
bd7614372a | ||
|
|
61b5aa4900 | ||
|
|
a6e75b46cb | ||
|
|
dd32c7f9b2 | ||
|
|
aaa5c82236 | ||
|
|
8b68c103c5 | ||
|
|
454f94fc54 | ||
|
|
8b7d380de1 | ||
|
|
8052a4287e | ||
|
|
f114b13306 | ||
|
|
ce70728eeb | ||
|
|
8518023b0b | ||
|
|
33988ac84c |
No files matched your search
+8
-5
@@ -2,7 +2,7 @@
|
||||
|
||||
# The last matching rule takes precedence.
|
||||
|
||||
- @Salazareo
|
||||
* @Salazareo
|
||||
|
||||
# GUI
|
||||
|
||||
@@ -18,7 +18,10 @@
|
||||
# Backend AI
|
||||
|
||||
/src/backend/controllers/puterai/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/ai-\*/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/ai/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/util/aiCostFactor* @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/util/aiPrices* @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/util/aiProviderKey* @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# Backend: AI, hosting, workers, networking, and WebDAV
|
||||
|
||||
@@ -29,7 +32,7 @@
|
||||
/src/backend/drivers/workers/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/services/localworker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/stores/subdomain/ @ProgrammerIn-wonderland @Salazareo
|
||||
/extensions/workerSandbox\* @ProgrammerIn-wonderland @Salazareo
|
||||
/extensions/workerSandbox* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker-types/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/peer/ @ProgrammerIn-wonderland @Salazareo
|
||||
@@ -42,7 +45,7 @@
|
||||
/src/backend/services/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/acl/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/apps/AppPermissionService\* @jfcastro92 @Salazareo
|
||||
/src/backend/services/apps/AppPermissionService* @jfcastro92 @Salazareo
|
||||
/src/backend/stores/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/group/ @jfcastro92 @Salazareo
|
||||
@@ -59,7 +62,7 @@
|
||||
/src/puter-js/tests/api/suites/hosting.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/workers.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/net.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Peer\* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Peer* @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# SDK: sharing and permissions
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# This file is machine-generated by `gh actions-lock`.
|
||||
# Do not edit by hand; run `gh actions-lock` to update.
|
||||
# Docs: https://gh.io/actions-lockfile
|
||||
version: 'v0.0.3'
|
||||
workflows:
|
||||
'.github/workflows/ai-provider-integration-tests.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
'.github/workflows/backend-tests.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'davelosert/[email protected]'
|
||||
- 'dorny/[email protected]'
|
||||
'.github/workflows/dependabot-automerge.yml':
|
||||
- 'dependabot/[email protected]'
|
||||
'.github/workflows/docker-image.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'peter-evans/[email protected]'
|
||||
'.github/workflows/docs-build.yml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
'.github/workflows/notify-dev.yaml': []
|
||||
'.github/workflows/notify-prod.yaml': []
|
||||
'.github/workflows/puterjs-tests.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'davelosert/[email protected]'
|
||||
- 'dorny/[email protected]'
|
||||
'.github/workflows/tag-alias.yaml':
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
dependencies:
|
||||
'actions/[email protected]':
|
||||
ref: 'v7.0.1'
|
||||
commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
|
||||
owner_id: 44036562
|
||||
repo_id: 197814629
|
||||
'actions/[email protected]':
|
||||
ref: 'v8.0.2'
|
||||
commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
|
||||
owner_id: 44036562
|
||||
repo_id: 192626254
|
||||
'actions/[email protected]':
|
||||
ref: 'v7.1.0'
|
||||
commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
|
||||
owner_id: 44036562
|
||||
repo_id: 189476904
|
||||
'actions/[email protected]':
|
||||
ref: 'v7.0.2'
|
||||
commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
|
||||
owner_id: 44036562
|
||||
repo_id: 192625955
|
||||
'davelosert/[email protected]':
|
||||
ref: 'v2.13.2'
|
||||
commit: 'sha1-74c2d9210b4fc44c84d8213f61d7ec99837b7aa0'
|
||||
owner_id: 4287128
|
||||
repo_id: 501176456
|
||||
'dependabot/[email protected]':
|
||||
ref: 'v3.1.0'
|
||||
commit: 'sha1-25dd0e34f4fe68f24cc83900b1fe3fe149efef98'
|
||||
owner_id: 27347476
|
||||
repo_id: 371068214
|
||||
'docker/[email protected]':
|
||||
ref: 'v7.4.0'
|
||||
commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc'
|
||||
owner_id: 5429470
|
||||
repo_id: 241092383
|
||||
'docker/[email protected]':
|
||||
ref: 'v4.8.0'
|
||||
commit: 'sha1-8aa5831fb8d48d8f76594c7ddde61ae58019f550'
|
||||
owner_id: 5429470
|
||||
repo_id: 287743349
|
||||
'docker/[email protected]':
|
||||
ref: 'v6.2.0'
|
||||
commit: 'sha1-dc802804100637a589fabce1cb79ff13a1411302'
|
||||
owner_id: 5429470
|
||||
repo_id: 306769011
|
||||
'docker/[email protected]':
|
||||
ref: 'v4.4.1'
|
||||
commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
|
||||
owner_id: 5429470
|
||||
repo_id: 288485773
|
||||
'docker/[email protected]':
|
||||
ref: 'v4.4.0'
|
||||
commit: 'sha1-99012661954931238ded8c8b007157a8430204e1'
|
||||
owner_id: 5429470
|
||||
repo_id: 288487894
|
||||
'dorny/[email protected]':
|
||||
ref: 'v4.0.3'
|
||||
commit: 'sha1-ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d'
|
||||
owner_id: 1588572
|
||||
repo_id: 265601040
|
||||
'peter-evans/[email protected]':
|
||||
ref: 'v5.0.0'
|
||||
commit: 'sha1-1b9a80c056b620d92cedb9d9b5a223409c68ddfa'
|
||||
owner_id: 18365890
|
||||
repo_id: 193685475
|
||||
@@ -17,10 +17,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -29,10 +31,9 @@ jobs:
|
||||
run: npm ci
|
||||
|
||||
- name: Run AI provider integration tests
|
||||
# Vitest treats positional args as filename substring filters.
|
||||
# The pattern matches every `*.integration.test.ts` file and
|
||||
# nothing else — the regular backend test suite stays out.
|
||||
run: npm run test:backend -- integration.test
|
||||
# Only the provider tests: the other `*.integration.test.ts`
|
||||
# files need no credentials and run with the backend suite.
|
||||
run: npm run test:backend -- $(git ls-files 'src/backend/drivers/ai/**/*.integration.test.ts')
|
||||
env:
|
||||
CI: 'true'
|
||||
PUTER_TEST_AI_CLAUDE_API_KEY: ${{ secrets.PUTER_TEST_AI_CLAUDE_API_KEY }}
|
||||
@@ -47,6 +48,15 @@ jobs:
|
||||
PUTER_TEST_AI_MOONSHOT_API_KEY: ${{ secrets.PUTER_TEST_AI_MOONSHOT_API_KEY }}
|
||||
PUTER_TEST_AI_ZAI_API_KEY: ${{ secrets.PUTER_TEST_AI_ZAI_API_KEY }}
|
||||
PUTER_TEST_AI_ELEVENLABS_API_KEY: ${{ secrets.PUTER_TEST_AI_ELEVENLABS_API_KEY }}
|
||||
PUTER_TEST_AI_ALIBABA_API_KEY: ${{ secrets.PUTER_TEST_AI_ALIBABA_API_KEY }}
|
||||
PUTER_TEST_AI_AZURE_OPENAI_API_KEY: ${{ secrets.PUTER_TEST_AI_AZURE_OPENAI_API_KEY }}
|
||||
PUTER_TEST_AI_AZURE_OPENAI_API_URL: ${{ secrets.PUTER_TEST_AI_AZURE_OPENAI_API_URL }}
|
||||
PUTER_TEST_AI_BYTEPLUS_API_KEY: ${{ secrets.PUTER_TEST_AI_BYTEPLUS_API_KEY }}
|
||||
PUTER_TEST_AI_HOONIFY_API_KEY: ${{ secrets.PUTER_TEST_AI_HOONIFY_API_KEY }}
|
||||
PUTER_TEST_AI_INFRON_API_KEY: ${{ secrets.PUTER_TEST_AI_INFRON_API_KEY }}
|
||||
PUTER_TEST_AI_KLING_API_KEY: ${{ secrets.PUTER_TEST_AI_KLING_API_KEY }}
|
||||
PUTER_TEST_AI_META_API_KEY: ${{ secrets.PUTER_TEST_AI_META_API_KEY }}
|
||||
PUTER_TEST_AI_MINIMAX_API_KEY: ${{ secrets.PUTER_TEST_AI_MINIMAX_API_KEY }}
|
||||
|
||||
# Realtime conversations through the SDK against live OpenAI. Builds
|
||||
# the SDK bundle these tests load itself, so it runs even when the
|
||||
|
||||
@@ -23,8 +23,10 @@ jobs:
|
||||
outputs:
|
||||
backend: ${{ steps.filter.outputs.backend }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: dorny/paths-filter@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dorny/[email protected]
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
@@ -52,10 +54,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -80,13 +84,14 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -106,7 +111,7 @@ jobs:
|
||||
|
||||
- name: Upload blob report
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: backend-blob-${{ matrix.shard }}
|
||||
path: .vitest-reports/
|
||||
@@ -146,13 +151,14 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -161,7 +167,7 @@ jobs:
|
||||
run: npm ci
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
pattern: backend-blob-*
|
||||
path: .vitest-reports
|
||||
@@ -173,7 +179,7 @@ jobs:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload coverage artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: backend-coverage-${{ github.event_name == 'push' && 'main' || 'pr' }}
|
||||
path: src/backend/coverage/
|
||||
@@ -204,7 +210,7 @@ jobs:
|
||||
|
||||
- name: Report coverage on PR
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: davelosert/vitest-coverage-report-action@v2
|
||||
uses: davelosert/vitest-coverage-report-action@v2.13.2
|
||||
with:
|
||||
json-summary-path: src/backend/coverage/coverage-summary.json
|
||||
json-final-path: src/backend/coverage/coverage-final.json
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
steps:
|
||||
- name: Dependabot metadata
|
||||
id: metadata
|
||||
uses: dependabot/fetch-metadata@v3
|
||||
uses: dependabot/fetch-metadata@v3.1.0
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
|
||||
@@ -45,30 +45,32 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Log in to GitHub Package Container registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v6.2.0
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
@@ -82,7 +84,7 @@ jobs:
|
||||
type=raw,value=main,enable=${{ github.event_name == 'push' || inputs.push_main }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v7.4.0
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
@@ -95,7 +97,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Sync README to Docker Hub
|
||||
uses: peter-evans/dockerhub-description@v5
|
||||
uses: peter-evans/dockerhub-description@v5.0.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
@@ -5,16 +5,21 @@ on:
|
||||
paths:
|
||||
- 'src/docs/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-docs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Maintain Release Merge PR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
update-release-pr:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Setup Git
|
||||
run: |
|
||||
git config user.name github-actions
|
||||
git config user.email [email protected]
|
||||
- name: Check for existing PR
|
||||
id: find-pr
|
||||
uses: juliangruber/find-pull-request-action@v1
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
branch: release
|
||||
- uses: actions/checkout@v7
|
||||
if: steps.find-pr.outputs.number == ''
|
||||
with:
|
||||
ref: release
|
||||
- name: Reset release branch
|
||||
if: steps.find-pr.outputs.number == ''
|
||||
run: |
|
||||
git fetch origin main:main
|
||||
git reset --hard main
|
||||
- name: Create/Update PR
|
||||
if: steps.find-pr.outputs.number == ''
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
commit-message: Update release branch
|
||||
title: Merge Release Auto-PR
|
||||
body: Merging this PR will invoke release actions
|
||||
branch: auto-update/release
|
||||
@@ -5,6 +5,8 @@ on:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -22,8 +22,10 @@ jobs:
|
||||
outputs:
|
||||
puterjs: ${{ steps.filter.outputs.puterjs }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: dorny/paths-filter@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dorny/[email protected]
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
@@ -52,10 +54,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -73,10 +77,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -130,14 +136,15 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout ${{ matrix.ref }}
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ matrix.ref }}
|
||||
repository: ${{ matrix.repository }}
|
||||
|
||||
- name: Set up Node.js
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
@@ -161,7 +168,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: needs.changes.outputs.puterjs == 'true' && !cancelled()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: puterjs-coverage-${{ matrix.artifact }}
|
||||
path: src/puter-js/coverage/coverage-*.json
|
||||
@@ -174,13 +181,15 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Missing when the PR leg failed; `coverage (pr)` already reports that.
|
||||
- name: Download PR coverage
|
||||
id: pr-coverage
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
name: puterjs-coverage-pr
|
||||
path: src/puter-js/coverage
|
||||
@@ -191,14 +200,14 @@ jobs:
|
||||
id: base-coverage
|
||||
if: steps.pr-coverage.outcome == 'success'
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
name: puterjs-coverage-base
|
||||
path: coverage-base
|
||||
|
||||
- name: Report coverage on PR
|
||||
if: steps.pr-coverage.outcome == 'success' && steps.base-coverage.outcome == 'success'
|
||||
uses: davelosert/vitest-coverage-report-action@v2
|
||||
uses: davelosert/vitest-coverage-report-action@v2.13.2
|
||||
with:
|
||||
name: puter.js SDK
|
||||
json-summary-path: src/puter-js/coverage/coverage-summary.json
|
||||
@@ -207,7 +216,7 @@ jobs:
|
||||
|
||||
- name: Report coverage on PR (no base comparison)
|
||||
if: steps.pr-coverage.outcome == 'success' && steps.base-coverage.outcome != 'success'
|
||||
uses: davelosert/vitest-coverage-report-action@v2
|
||||
uses: davelosert/vitest-coverage-report-action@v2.13.2
|
||||
with:
|
||||
name: puter.js SDK
|
||||
json-summary-path: src/puter-js/coverage/coverage-summary.json
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
name: release-please
|
||||
|
||||
jobs:
|
||||
release-please:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: google-github-actions/release-please-action@v4
|
||||
with:
|
||||
release-type: node
|
||||
@@ -45,14 +45,14 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Log in to GitHub Package Container registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
|
||||
- name: Create alias tag
|
||||
env:
|
||||
|
||||
@@ -9,6 +9,7 @@ Use these as the source of truth before exploring further:
|
||||
- [README.md](README.md) — project overview and quickstart.
|
||||
- [doc/architecture.md](doc/architecture.md) — backend layered stack (controllers → drivers → services → stores → clients), `PuterServer` wiring, `Context` (ALS), actors, and extensions.
|
||||
- [doc/contributing-apis.md](doc/contributing-apis.md) — adding and maintaining public APIs end to end (backend surface → puter.js → types → docs → tests). Follow it for any API work.
|
||||
- [doc/ai-drivers.md](doc/ai-drivers.md) — where AI driver code lives (interfaces, providers, protocols, shared), the import rules between those folders, and how to add a provider or an AI interface.
|
||||
- [doc/pagination.md](doc/pagination.md) — the one pagination convention for list APIs (limit/cursor/offset/includeTotal, envelope shape, cursor semantics).
|
||||
- [doc/alarms.md](doc/alarms.md) — raising alarms and picking a severity (what pages, what only gets recorded), plus the config that routes them.
|
||||
- [doc/self-hosting.md](doc/self-hosting.md) — running Puter outside hosted infra.
|
||||
@@ -70,7 +71,7 @@ Commit messages and PR descriptions never credit anyone or mention a bug report.
|
||||
- **Test new behavior.** Every new function, endpoint, driver method, or logic branch gets a test; every bug fix gets a regression test that fails before the fix. If something is genuinely hard to test, skip it but say so in the PR.
|
||||
- **Boy Scout Rule, proportional to the change.** In files you're already touching, fix the typo, the dead import, the missing test, the bit you had to read twice. Don't ride a refactor along with a bug fix.
|
||||
- **Understand what you commit.** AI assistance is fine; shipping code you couldn't have written, debugged, or defended in review is not.
|
||||
- **A limit change is not done until the docs change with it.** Every rate limit, concurrency cap, quota, and allowance in the code is published in [src/docs/src/rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) — an undisclosed limit is one developers discover as a service failure. If a PR moves any of these numbers, the same PR updates that page. The numbers live in `src/backend/controllers/fs/limits.ts` (filesystem), `src/backend/drivers/util/aiLimits.ts` (all AI drivers), per-driver `rateLimit`/`concurrent` configs, and `src/backend/data/subPolicies/` (free-tier allowances).
|
||||
- **A limit change is not done until the docs change with it.** Every rate limit, concurrency cap, quota, and allowance in the code is published in [src/docs/src/rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) — an undisclosed limit is one developers discover as a service failure. If a PR moves any of these numbers, the same PR updates that page. The numbers live in `src/backend/controllers/fs/limits.ts` (filesystem), `src/backend/drivers/ai/shared/aiLimits.ts` (all AI drivers), per-driver `rateLimit`/`concurrent` configs, and `src/backend/data/subPolicies/` (free-tier allowances).
|
||||
|
||||
---
|
||||
|
||||
@@ -97,6 +98,10 @@ Layer rules:
|
||||
- **Drivers and controllers stay thin.** Domain rules a second entry point needs (signup, app CRUD, worker deploy, AI routing/billing) live in a service, not in the first driver or controller that needed them.
|
||||
- **Prefer explicit arguments over [`Context`](src/backend/core/context.ts) (ALS).** Use it only for request-scoped values that would otherwise thread through many layers; today that's mostly `actor` and `req`.
|
||||
|
||||
### AI drivers
|
||||
|
||||
AI drivers live in [src/backend/drivers/ai/](src/backend/drivers/ai/): `interfaces/` holds the Puter-facing drivers, `providers/<vendor>/<interface>/` holds vendor code, `protocols/` holds wire protocols several vendors speak, and `shared/` holds helpers more than one interface uses. Follow [doc/ai-drivers.md](doc/ai-drivers.md) when adding or moving AI code.
|
||||
|
||||
### Extensions
|
||||
|
||||
[extensions/](extensions/) is for **non-crucial parts of the system**: Puter must still work with the extension removed. If core needs to call it, it belongs in core ([whoami](extensions/whoami.ts) is the cautionary example). Inside an extension, follow the same layered structure, unless it only needs a few route handlers, in which case the `extension.get/post/...` helpers are enough on their own. The `extension` API is in [doc/architecture.md](doc/architecture.md#extension-api).
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ The following have already been reviewed and determined **not to be vulnerabilit
|
||||
- **SSRF via `secureFetch`.** Production routes outbound requests through an isolated proxy that has no access to internal/SSRF-sensitive resources.
|
||||
- **Attacks that depend on guessing an `appInstanceID` or app UID.** These are random 128-bit secret values and are not considered guessable.
|
||||
- **Apps invoking drivers, creating workers, or using KV.** Applications are intended to do this; worker permissions are scoped to the owning app. This is by design.
|
||||
- **App metadata or app user-count "leaks".** This information is currently public by design. This covers **private apps** too: private apps are an unreleased, admin-only beta for paid listings in the Puter App Center. Their `index_url` and other metadata are not secret, the flag is not user-settable, and no production app relies on it. Reports that a private app's metadata is reachable through a sibling route (taskbar items, `/get-launch-apps`, `index_url` drift on the hosting gate, etc.) are normalization work, not vulnerabilities.
|
||||
- **App metadata or app user-count "leaks".** This information is currently public by design. This covers **private apps** too: private apps are an unreleased, admin-only beta for paid listings in the Puter App Center. Their `index_url` and other metadata are not secret, the flag is not user-settable, and no production app relies on it. Reports that a private app's metadata is reachable through a sibling route (`/get-launch-apps`, `index_url` drift on the hosting gate, etc.) are normalization work, not vulnerabilities.
|
||||
- **Worker hot-reload driven by a share recipient.** Sharing a worker's source file with write access is how you let a collaborator change that worker's code and have it deploy live. The redeploy running as the file owner, with the owner's worker credentials, is the intended behavior of that grant — the owner chose to hand out write access to executable code. If you do not want that, do not share the source file with write access.
|
||||
- **Re-claiming a site or worker name its owner deleted.** A hosted site or worker name that its owner deletes is treated as abandoned by that owner. If one of their apps still points at it, whoever claims the name next serves that app's `index_url` and receives what the app is launched with, and a new app on the same origin may share the old one's identity. Guarding app identities against being recycled this way is ongoing hardening, not a security boundary, so new ways to re-claim an abandoned name, or to inherit an app's tokens or data through one, are not eligible.
|
||||
- **Share-mode downgrades that do not cascade.** Demoting a delegate does not retroactively revoke the grants that delegate already issued downstream. Shares are independent grants once issued, not a live-evaluated chain. Revoking the downstream grant is a separate, supported action.
|
||||
|
||||
@@ -254,6 +254,16 @@
|
||||
"user": "",
|
||||
"pass": ""
|
||||
}
|
||||
// Each send names a stream: "auth", "signup" or "notify". A stream
|
||||
// listed here adds its headers to every message on it, and its "from"
|
||||
// when the sender gives none. Tags are written to "tagsHeader".
|
||||
// "streams": {
|
||||
// "notify": {
|
||||
// "from": "\"Puter\" <[email protected]>",
|
||||
// "headers": { "X-Example-Stream": "notify" },
|
||||
// "tagsHeader": "X-Example-Tags"
|
||||
// }
|
||||
// }
|
||||
},
|
||||
|
||||
// ── Inbound mail (separate process) ─────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
# AI Drivers
|
||||
|
||||
Where the AI driver code lives under [src/backend/drivers/ai/](../src/backend/drivers/ai/), what each folder owns, and how to add a provider or a new AI interface. The general layer rules are in [architecture.md](architecture.md) and [AGENTS.md](../AGENTS.md#backend). This page covers what is specific to AI.
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
src/backend/drivers/ai/
|
||||
├── interfaces/ Puter-facing drivers, one folder per AI interface
|
||||
├── providers/ Vendor code, as providers/<vendor>/<interface>/
|
||||
├── protocols/ Wire protocols that several vendors speak
|
||||
└── shared/ Helpers that more than one interface uses
|
||||
```
|
||||
|
||||
| Folder | Owns | Example |
|
||||
| --- | --- | --- |
|
||||
| `interfaces/<interface>/` | The driver class, the interface's contracts in `types.ts`, its provider base class where one exists, and the helpers and tests that apply to every provider of that interface. | [interfaces/chat/ChatCompletionDriver.ts](../src/backend/drivers/ai/interfaces/chat/ChatCompletionDriver.ts) |
|
||||
| `providers/<vendor>/<interface>/` | One vendor's implementation of one interface: the adapter, its model catalog and prices, its fixtures, and its tests. | [providers/gemini/video/](../src/backend/drivers/ai/providers/gemini/video/) |
|
||||
| `providers/<vendor>/` | Helpers that several of that vendor's interfaces use. | [providers/gemini/geminiRest.ts](../src/backend/drivers/ai/providers/gemini/geminiRest.ts) |
|
||||
| `protocols/openaiCompatible/` | The OpenAI wire protocol: the base provider class, message and usage conversion, and parameter mapping for Chat Completions and Responses. | [protocols/openaiCompatible/OpenAICompatProvider.ts](../src/backend/drivers/ai/protocols/openaiCompatible/OpenAICompatProvider.ts) |
|
||||
| `shared/` | Helpers that more than one interface uses: limits, the provider registry, the remote catalog cache, job polling, streamed charging, and media input. | [shared/aiLimits.ts](../src/backend/drivers/ai/shared/aiLimits.ts) |
|
||||
|
||||
Helpers that non-AI code also uses stay in [src/backend/drivers/util/](../src/backend/drivers/util/): file input, data URIs, upstream error mapping, and stream piping.
|
||||
|
||||
AI code that a service also needs lives in [src/backend/util/](../src/backend/util/), because a service can't import from the driver layer. The realtime relay in `services/realtime/` bills through the same cost factor as the drivers, so these three files are there:
|
||||
|
||||
| File | Holds |
|
||||
| --- | --- |
|
||||
| [aiCostFactor.ts](../src/backend/util/aiCostFactor.ts) | `withAiCostFactor`, the metering view every AI driver and the realtime relay record usage through, and `withAiCreditHold`. |
|
||||
| [aiProviderKey.ts](../src/backend/util/aiProviderKey.ts) | `readProviderKey`, which reads an API key from `config.providers` blocks. |
|
||||
| [aiPrices.ts](../src/backend/util/aiPrices.ts) | Vendor prices that a driver and a service both bill. |
|
||||
|
||||
### Interfaces
|
||||
|
||||
| Folder | Driver | `driverInterface` | `driverName` | Key in `puterDrivers` |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `interfaces/chat/` | `ChatCompletionDriver` | `puter-chat-completion` | `ai-chat` | `aiChat` |
|
||||
| `interfaces/image/` | `ImageGenerationDriver` | `puter-image-generation` | `ai-image` | `aiImage` |
|
||||
| `interfaces/video/` | `VideoGenerationDriver` | `puter-video-generation` | `ai-video` | `aiVideo` |
|
||||
| `interfaces/textToSpeech/` | `TTSDriver` | `puter-tts` | `ai-tts` | `aiTts` |
|
||||
| `interfaces/speechToText/` | `SpeechToTextDriver` | `puter-speech2txt` | `ai-speech2txt` | `aiSpeech2Txt` |
|
||||
| `interfaces/speechToSpeech/` | `VoiceChangerDriver` | `puter-speech2speech` | `ai-speech2speech` | `aiSpeech2Speech` |
|
||||
| `interfaces/ocr/` | `OCRDriver` | `puter-ocr` | `ai-ocr` | `aiOcr` |
|
||||
|
||||
Folder names are not runtime identifiers. `driverInterface`, `driverName`, driver aliases, the keys under `providers` in config, provider ids, model ids, and usage keys are public contracts. Moving or renaming a file never changes them.
|
||||
|
||||
## Request flow
|
||||
|
||||
A call goes from the interface to a provider, and from there to a shared protocol when the vendor speaks one:
|
||||
|
||||
```text
|
||||
interfaces/chat/ChatCompletionDriver.ts
|
||||
validates the call, picks the model and provider, holds credits
|
||||
→ providers/deepseek/chat/DeepSeekProvider.ts
|
||||
DeepSeek's catalog, endpoint, and request differences
|
||||
→ protocols/openaiCompatible/OpenAICompatProvider.ts
|
||||
the OpenAI-compatible request, response, and stream handling
|
||||
```
|
||||
|
||||
A provider whose vendor has its own API, such as `ClaudeProvider` or `ElevenLabsTTSProvider`, implements the interface's contract directly and uses no protocol.
|
||||
|
||||
## Import rules
|
||||
|
||||
- `shared/` imports nothing from `interfaces/`, `providers/`, or `protocols/`.
|
||||
- Nothing in `services/`, `stores/`, or `clients/` imports from `drivers/ai/`. Code that a service and a driver both need goes in `src/backend/util/`.
|
||||
- `providers/` and `protocols/` import an interface's `types.ts` and helpers. They never import a driver class. Their tests can, to drive a provider through the real driver.
|
||||
- Only an interface's driver file imports providers, to register them. The interface's helpers don't.
|
||||
- An interface doesn't import another interface. When a second interface needs a helper, move the helper to `shared/`, or to `providers/<vendor>/` when it belongs to one vendor.
|
||||
- A vendor folder imports another vendor's folder only when it serves that vendor's models. `providers/azure/` reads the OpenAI and xAI catalogs for that reason.
|
||||
|
||||
## Where a file goes
|
||||
|
||||
| The code is | It goes in |
|
||||
| --- | --- |
|
||||
| Input validation, gates, routing, credit holds, or anything else every provider of the interface gets | `interfaces/<interface>/` |
|
||||
| A request or response contract that providers implement | `interfaces/<interface>/types.ts` |
|
||||
| A vendor's endpoint, request shape, catalog, prices, or quirks | `providers/<vendor>/<interface>/` |
|
||||
| A vendor's auth, REST plumbing, or constants that two of its interfaces use | `providers/<vendor>/` |
|
||||
| Behavior of a wire protocol that several vendors speak | `protocols/<protocol>/` |
|
||||
| A helper that two or more interfaces use and that belongs to no vendor | `shared/` |
|
||||
| A helper that non-AI drivers or controllers also use | `src/backend/drivers/util/` |
|
||||
| AI code that a service also uses | `src/backend/util/`, in a file named `ai<Something>.ts` |
|
||||
|
||||
Keep a helper next to its only consumer. Move it up when a second consumer appears, not before.
|
||||
|
||||
When one Puter option becomes a different request shape per vendor, the interface reads the option and each dialect maps it. The chat `compaction` option is read in `interfaces/chat/utils/compaction.js`, mapped for OpenAI by `toOpenAiContextManagement` in `protocols/openaiCompatible/openaiParams.ts`, and mapped for Anthropic by `toAnthropicContextManagement` in `providers/claude/chat/contextManagement.ts`. Tools work the same way: `normalize_tools_object` in the interface, then `toolsToWire` for OpenAI and `claudeToolPolicy` for Anthropic.
|
||||
|
||||
Tests sit next to the code they test. Fixtures go in a `fixtures/` folder next to the test that reads them.
|
||||
|
||||
## Adding a provider
|
||||
|
||||
1. Create `providers/<vendor>/<interface>/`. Use the vendor's existing folder if it has one, so all of a vendor's interfaces stay together.
|
||||
2. Implement the interface's contract from `interfaces/<interface>/types.ts`:
|
||||
|
||||
| Interface | Contract | Base class |
|
||||
| --- | --- | --- |
|
||||
| chat | `IChatProvider` | `OpenAICompatProvider` for an OpenAI-compatible vendor |
|
||||
| image | `IImageProvider` | none |
|
||||
| video | `IVideoProvider` | `VideoProvider` |
|
||||
| textToSpeech | `ITTSProvider` | none |
|
||||
| speechToText | `ISpeechToTextProvider` | `SpeechToTextProvider` |
|
||||
| ocr | `IOcrProvider` | none |
|
||||
|
||||
Speech to speech has one provider, so `VoiceChangerDriver` uses `ElevenLabsSpeechToSpeechProvider` directly. Add a contract to its `types.ts` when a second provider needs one.
|
||||
|
||||
3. Put the model catalog and prices in the same folder, as `models.ts` or `costs.ts`. A provider that loads its catalog from the vendor wraps the fetch in `cachedRemoteCatalog` from `shared/`.
|
||||
4. Register the provider in the interface's driver: in `#registerProviders()`, or in `onServerStart()` for OCR and speech to speech. A chat provider built from an API key and an optional base URL is one row in `KEYED_PROVIDERS` instead. Text to speech and speech to text also list the provider id and its aliases in `providerAliases.ts`.
|
||||
5. Read credentials from `config.providers[<key>]`. The key is part of the self-hosting config contract, so reuse the vendor's existing key when it has one.
|
||||
6. Add tests next to the provider. Mock the vendor SDK or `fetch` at the network boundary and drive the provider through the real driver where you can. A live test is a `*.integration.test.ts` file that skips itself, with `skipUnlessEnv` from [testEnv.ts](../src/backend/testEnv.ts), unless its `PUTER_TEST_AI_*` variable is set. Helpers for provider tests are in [shared/integrationTestUtil.ts](../src/backend/drivers/ai/shared/integrationTestUtil.ts).
|
||||
7. For a chat provider, add its static catalog to `CATALOGS` in [modelCatalogs.test.ts](../src/backend/drivers/ai/interfaces/chat/modelCatalogs.test.ts) and the provider to [providerConsistency.test.ts](../src/backend/drivers/ai/interfaces/chat/providerConsistency.test.ts). The catalog test fails when a `providers/*/chat/` file with "model" in its name exports a catalog that isn't listed.
|
||||
8. Add the provider's config key to the `providers` config of the interface's [fuzz suite](#fuzz-tests), and its id to the list of providers the suite expects.
|
||||
9. Update the published docs for the models and limits the provider adds, following [contributing-apis.md](contributing-apis.md).
|
||||
|
||||
### OpenAI-compatible vendors
|
||||
|
||||
A vendor that speaks the OpenAI Chat Completions protocol extends `OpenAICompatProvider` and overrides only what differs. It still gets its own `providers/<vendor>/chat/` folder.
|
||||
|
||||
Vendors share one constants-only folder only when at least six of them differ in nothing but base URL and model catalog. Each then becomes one data file under `providers/openaiCompatible/providers/`, read by a single implementation. No vendor qualifies today, because every subclass overrides request, catalog, or billing behavior.
|
||||
|
||||
## Fuzz tests
|
||||
|
||||
Two seeded suites send malformed input through the real drivers and providers. They run with the rest of the backend tests.
|
||||
|
||||
| Suite | Covers |
|
||||
| --- | --- |
|
||||
| [interfaces/chat/ChatCompletionDriver.fuzz.test.ts](../src/backend/drivers/ai/interfaces/chat/ChatCompletionDriver.fuzz.test.ts) | Chat requests with a few values changed, and valid requests against wrong statuses, bodies of the wrong shape, and broken event streams. |
|
||||
| [interfaces/mediaDrivers.fuzz.test.ts](../src/backend/drivers/ai/interfaces/mediaDrivers.fuzz.test.ts) | Image, video, speech and OCR requests with a few values changed. Every vendor refuses the request. |
|
||||
|
||||
What the suites assert:
|
||||
|
||||
- A malformed request never fails with an error that has no status, which `/drivers/call` would report as an unclassified 500.
|
||||
- For chat, with the vendors answering, a malformed request also never fails with a 5xx and never marks a route unhealthy.
|
||||
- When the upstream misbehaves, chat ends with an answer or an `HttpError`. A stream only has to end: the error line a stream carries isn't checked, so a malformed event can still end one with a 500.
|
||||
- Requests reach the vendors, and with the vendors answering some chat requests are answered, so a suite that stops exercising the providers fails.
|
||||
|
||||
The vendors are played at the HTTP boundary by `upstream` in [shared/fuzzTestUtil.ts](../src/backend/drivers/ai/shared/fuzzTestUtil.ts), which stands in for global `fetch`, undici `fetch` and axios, and makes `https.request` throw so an SDK with its own client fails the suite. The vendors' SDKs run, except Groq's, which is swapped for the OpenAI SDK, and the AWS clients, whose `send` is stubbed. The media suite calls the drivers as a user with unlimited metering, so a request gets past file input and the credit hold to the vendor. A provider that loads its catalog from the vendor registers no models there, so the suites skip it.
|
||||
|
||||
A failure lists each distinct finding with its case number and input. Each case is generated from the seed and its own number, so the same case fails again on the next run. Routes are picked by rendezvous hashing: a model added to a catalog takes only the cases it wins, and every other case keeps its route. To search further, raise the case count:
|
||||
|
||||
```sh
|
||||
PUTER_FUZZ_CASES=100000 npx vitest run --config src/backend/vitest.config.ts src/backend/drivers/ai/interfaces/chat/ChatCompletionDriver.fuzz.test.ts
|
||||
```
|
||||
|
||||
When a suite finds a crash, validate the value where the request enters, in the driver or in `normalize_single_message`, so every provider is covered. Reject only what crashed: a value a provider ignored or refused before must still reach it. An option a driver reads as text, a number or a flag goes in the driver's `SCALAR_OPTIONS`, and `assertScalarOptions` from `shared/scalarOptions.ts` answers 400 for a value there that can't be converted to text. Add a unit test for the case next to the fix.
|
||||
|
||||
## Adding an interface
|
||||
|
||||
1. Create `interfaces/<interface>/` with the driver class and a `types.ts` for the contracts providers implement.
|
||||
2. Register the driver in [src/backend/drivers/index.ts](../src/backend/drivers/index.ts).
|
||||
3. Use `AI_RATE_LIMIT` and `AI_CONCURRENT` from `shared/aiLimits.ts`, and publish any new limit in [rate-limits-and-quotas.md](../src/docs/src/rate-limits-and-quotas.md).
|
||||
4. Meter through `withAiCostFactor` from `src/backend/util/aiCostFactor.ts`, and route between providers with `ProviderRegistry` from `shared/providerRegistry.ts`.
|
||||
5. List the options the providers read as text, a number or a flag in a `SCALAR_OPTIONS` array, and call `assertScalarOptions` from `shared/scalarOptions.ts` first in each method. Add the driver to [mediaDrivers.fuzz.test.ts](../src/backend/drivers/ai/interfaces/mediaDrivers.fuzz.test.ts).
|
||||
6. Follow [contributing-apis.md](contributing-apis.md) for the puter.js method, types, docs, and tests.
|
||||
@@ -42,6 +42,19 @@ block-beta
|
||||
|
||||
Extensions sit alongside this stack and can register into any layer; see [Extensions](#extensions).
|
||||
|
||||
### AI drivers
|
||||
|
||||
The AI drivers have their own layout under [src/backend/drivers/ai/](../src/backend/drivers/ai/):
|
||||
|
||||
| Folder | Holds |
|
||||
| --- | --- |
|
||||
| `interfaces/` | The Puter-facing drivers, one folder per AI interface (chat, image, video, text to speech, speech to text, speech to speech, OCR). |
|
||||
| `providers/` | Vendor code, as `providers/<vendor>/<interface>/`. |
|
||||
| `protocols/` | Wire protocols that several vendors speak, such as the OpenAI-compatible one. |
|
||||
| `shared/` | Helpers that more than one interface uses. |
|
||||
|
||||
A call goes from an interface's driver to a provider, and from there to a shared protocol when the vendor speaks one. [ai-drivers.md](ai-drivers.md) has the ownership rules, the import rules, and the steps for adding a provider.
|
||||
|
||||
## Entry point: `PuterServer`
|
||||
|
||||
`PuterServer` is the bootstrap. It:
|
||||
|
||||
+6
-6
@@ -16,7 +16,7 @@ Core subscribes the browser to a push service only when the server also publishe
|
||||
|
||||
`uid` is the notification's uid, a later push with the same `group` replaces the one showing, and `target` and `data` reach the app when the user taps it.
|
||||
|
||||
When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so the page's push client registers the new subscription.
|
||||
When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` keeps the new one, or subscribes again with Puter's stored key when the browser gave none and the event, the receipt or an earlier mark names the subscription that went. It writes `puter:subscription-changed:<scope>` to the `puter-push` IndexedDB as `{ at, oldEndpoint }`, the old endpoint kept from an earlier mark no page handled yet, else taken from the event or from the receipt, forgets the receipt kept for that scope, and posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so an open page runs `helpers/pushSync.js` on that message, after any run already in flight, and a later load does the same through the marker.
|
||||
|
||||
## Receipts
|
||||
|
||||
@@ -59,9 +59,9 @@ Apps on their own origin do the same through `push-sw.js`, which they import int
|
||||
|
||||
```ts
|
||||
interface PushClient {
|
||||
enable(): Promise<{ deviceUid: string }>;
|
||||
enable(options?: { background?: boolean }): Promise<{ deviceUid: string }>;
|
||||
disable(): Promise<void>;
|
||||
status(): Promise<{
|
||||
status(options?: { cached?: boolean }): Promise<{
|
||||
permission: 'default' | 'granted' | 'denied';
|
||||
registered: boolean;
|
||||
deviceUid?: string;
|
||||
@@ -69,12 +69,12 @@ interface PushClient {
|
||||
}
|
||||
```
|
||||
|
||||
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app holds `notifications:push`; the browser does not prompt then. The built-in client runs `puter.notifications.enable()`, which subscribes with the `/sw.js` registration and registers the browser through the Web Push flow above (`platform: 'webpush'`). It then resolves the `uid` of this browser's `webpush` device from `listDevices()`. It rejects when permission is refused or the browser wasn't registered.
|
||||
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app may push to Puter's devices (`puterDevices` in `GET /push/apps`); the browser does not prompt then. That call passes `{ background: true }`: a browser the user removed in Puter stays removed, and the built-in client rejects with code `push_device_removed`, after which core records the browser as turned off. Before a background call the built-in client asks `POST /push/web/subscriptions/status` about every earlier endpoint it knows: the marker's `oldEndpoint` and the last endpoint the server answered `active` for (`puter.push.last-endpoint:<user uuid>` in `localStorage`). On a 429 it registers nothing and rejects with `too_many_requests`, which the sync reports as `later`, keeping the marker for the next run. When any answer is `removed`, it ends the current subscription in the browser, clears the marker and rejects with `push_device_removed`, so a subscription change or a dropped subscription never brings back a browser removed from another device. A click renews it as before. For a click the built-in client runs `puter.notifications.enable()`, which subscribes with the `/sw.js` registration and registers the browser through the Web Push flow above (`platform: 'webpush'`); in the background it runs the same registration without replacing a removed subscription or opening Puter's sign-in. It then resolves the `uid` of this browser's `webpush` device from `listDevices()` and clears the subscription-changed marker. It rejects when permission is refused or the browser wasn't registered.
|
||||
- `disable()` runs `puter.notifications.disable()`, which sends `DELETE /push/web/subscriptions { endpoint }` and ends the browser's subscription. The tab calls it for Turn off, and for Remove on the device marked as this browser. Remove on any other device is `DELETE /push/devices/:uid`.
|
||||
- `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user, and `deviceUid` is that device's `uid`.
|
||||
- `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user and `POST /push/web/subscriptions/status` answers `active` for the current subscription's endpoint, and `deviceUid` is that device's `uid`. A server that can't answer leaves the device match alone, and so does an `unknown` for an endpoint `enable()` registered in this page in the last 5 minutes, since that check can trail a registration. While the subscription-changed marker is set, `registered` is false. With `{ cached: true }` a client may answer with the last registered state it read in this page. Core passes it for the silent registration on app launch and for an app's `puter.notifications.status()`; the Notifications tab and the push dialogs read fresh. The built-in client keeps that state in memory for the same user and subscription for up to 5 minutes, from `enable()` or once the server answered `active` for the endpoint, and answers from it without either request. Reads of the same user and endpoint share one real check while it is in flight. It drops it on `disable()`, a failed `enable()`, an unregistered answer, a set subscription-changed marker, and the worker's `puter-push-subscription-changed` message, so a new page load always asks the server. Whether the calling app may push is checked on every call, against a `GET /push/apps` answer the page reuses for 30 seconds and drops on `enable()`, `disable()`, a grant, a revoke and a user change. When it can't be read, or leaves the app out, the grant alone counts.
|
||||
|
||||
After `enable()` resolves, the tab calls `POST /push/devices/:uid/test` and asks the user whether a notification showed up.
|
||||
|
||||
The tab also reads `GET /push/apps` and `GET`/`POST /push/preferences`.
|
||||
|
||||
Core keeps its own record of a browser the user turned off, `puter.push.browser-off:<user uuid>` in `localStorage`, written by Turn off and by Remove on this browser's device, and cleared when `enable()` resolves. The silent registration above skips a browser with that record, so a device removed here stays removed while one removed from another device, or a new sign-in in a browser that already allows notifications, registers again.
|
||||
Core keeps its own record of a browser the user turned off, `puter.push.browser-off:<user uuid>` in `localStorage`, written by Turn off, by Remove on this browser's device, and by a silent registration the server refuses with `push_device_removed`, and cleared when `enable()` resolves. The silent registration above skips a browser with that record, so a device removed here or from another device stays removed until the user turns it on from a click, while a new sign-in in a browser that already allows notifications registers again.
|
||||
@@ -326,6 +326,20 @@ Used for password resets, email confirmation, and notifications. Without it thos
|
||||
|
||||
To require email confirmation before login, also set `"strict_email_verification_required": true`.
|
||||
|
||||
Account and notification mail is sent on a named stream: `auth` (password, address and 2FA changes), `signup` (address confirmation; a `puter.signup.validate` listener may name another: lowercase letters, digits, `:`, `_` and `-`, up to 64 characters, starting with a letter or digit) and `notify` (share, app feedback and contact form mail). Add `"streams"` inside `"email"` to configure one, for instance to let your mail provider route or report each kind of mail separately:
|
||||
|
||||
```json
|
||||
"streams": {
|
||||
"notify": {
|
||||
"from": "\"Puter\" <[email protected]>",
|
||||
"headers": { "X-Example-Stream": "notify" },
|
||||
"tagsHeader": "X-Example-Tags"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`headers` are added to every message on the stream, `from` replaces the default sender, and `tagsHeader` names the header that carries a message's tags as `name=value` pairs joined by `, `. A stream that is not listed adds nothing: the message goes out with the default from address and the caller's headers. `streams` is never handed to the transport.
|
||||
|
||||
To read the mail yourself instead of sending it, point the transport at a local SMTP sink — [MailHog](https://github.com/mailhog/MailHog) catches everything and serves it at `http://localhost:8025`. Publish both ports; a container that only exposes them is unreachable from the host:
|
||||
|
||||
```sh
|
||||
|
||||
@@ -257,8 +257,21 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(body.created_ts).toBeGreaterThan(0);
|
||||
// `directories` is only sent to user actors — confirm it’s present.
|
||||
expect(body.directories).toBeDefined();
|
||||
// taskbar_items is only sent to user actors.
|
||||
expect(body).toHaveProperty('taskbar_items');
|
||||
});
|
||||
|
||||
it('leaves out taskbar items saved on the account', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.update(user.id as number, {
|
||||
taskbar_items: JSON.stringify([{ type: 'app', name: 'editor' }]),
|
||||
});
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
expect(captured.body).not.toHaveProperty('taskbar_items');
|
||||
});
|
||||
|
||||
it('leaves out a desktop background saved on the account', async () => {
|
||||
@@ -449,7 +462,7 @@ describe('whoami extension — handleWhoami', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('cannot add taskbar_items for an app actor, which core omits', async () => {
|
||||
it('cannot add directories for an app actor, which core omits', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
@@ -457,11 +470,11 @@ describe('whoami extension — handleWhoami', () => {
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
(_key, event) => {
|
||||
event.details.taskbar_items = [{ name: 'injected' }];
|
||||
event.details.directories = { '/injected': 'uuid' };
|
||||
},
|
||||
);
|
||||
|
||||
expect(body).not.toHaveProperty('taskbar_items');
|
||||
expect(body).not.toHaveProperty('directories');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -542,7 +555,7 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(body.otp_secret).toBeUndefined();
|
||||
expect(body.signup_ip).toBeUndefined();
|
||||
// Not just absent as a key — the values must not appear anywhere
|
||||
// in the payload (nested under metadata, taskbar items, …).
|
||||
// in the payload (nested under metadata, …).
|
||||
const serialized = JSON.stringify(body);
|
||||
expect(serialized).not.toContain('5551234567');
|
||||
expect(serialized).not.toContain('fp_ABC123');
|
||||
|
||||
+1
-11
@@ -1,6 +1,5 @@
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import { APP_ICON_SIZES } from '@heyputer/backend/src/util/appIcon.js';
|
||||
import {
|
||||
buildUserDetails,
|
||||
scrubSensitive,
|
||||
@@ -54,19 +53,10 @@ export const handleWhoami = async (
|
||||
return;
|
||||
}
|
||||
|
||||
const rawIconSize =
|
||||
typeof req.query?.icon_size === 'string'
|
||||
? Number(req.query.icon_size)
|
||||
: undefined;
|
||||
const iconSize =
|
||||
rawIconSize !== undefined && APP_ICON_SIZES.includes(rawIconSize)
|
||||
? rawIconSize
|
||||
: undefined;
|
||||
|
||||
const details = await buildUserDetails(
|
||||
user,
|
||||
{ config: extension.config, clients, stores, services },
|
||||
{ isUser, iconSize, noIcons: !iconSize },
|
||||
{ isUser },
|
||||
);
|
||||
details.human_readable_age = user.timestamp
|
||||
? timeago.format(new Date(user.timestamp as string))
|
||||
|
||||
@@ -190,6 +190,13 @@ S3_BUCKET=puter-local
|
||||
providers = [ordered]@{
|
||||
ollama = [ordered]@{ enabled = $false }
|
||||
}
|
||||
# Off because a self-hosted install has no paid plans: every account
|
||||
# is on the free policy, so the plan gates would 402 the surfaces that
|
||||
# declare one (the OpenAI/Anthropic-compatible AI endpoints) for
|
||||
# everybody. Same as install.sh.
|
||||
meteringEnforcement = [ordered]@{
|
||||
subscriptions = $false
|
||||
}
|
||||
trust_proxy = 1
|
||||
}
|
||||
$configJson = $config | ConvertTo-Json -Depth 10
|
||||
|
||||
Generated
+86
-77
@@ -897,21 +897,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/core": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz",
|
||||
"integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.13.tgz",
|
||||
"integrity": "sha512-j1nvOQCZYSgp7HLmisnCXFTpVvFODiijQ9QHkORfGKWH6YDtskTEeCJS7eV545GZ9UBzrq0e2xsOd5QQ5PjIqw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.29.7",
|
||||
"@babel/generator": "^7.29.7",
|
||||
"@babel/helper-compilation-targets": "^7.29.7",
|
||||
"@babel/helper-module-transforms": "^7.29.7",
|
||||
"@babel/helpers": "^7.29.7",
|
||||
"@babel/parser": "^7.29.7",
|
||||
"@babel/generator": "^7.29.13",
|
||||
"@babel/helper-compilation-targets": "^7.29.13",
|
||||
"@babel/helper-module-transforms": "^7.29.13",
|
||||
"@babel/helpers": "^7.29.13",
|
||||
"@babel/parser": "^7.29.13",
|
||||
"@babel/template": "^7.29.7",
|
||||
"@babel/traverse": "^7.29.7",
|
||||
"@babel/types": "^7.29.7",
|
||||
"@babel/traverse": "^7.29.13",
|
||||
"@babel/types": "^7.29.11",
|
||||
"@jridgewell/remapping": "^2.3.5",
|
||||
"convert-source-map": "^2.0.0",
|
||||
"debug": "^4.1.0",
|
||||
@@ -938,14 +938,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/generator": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz",
|
||||
"integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.13.tgz",
|
||||
"integrity": "sha512-T5FRo+RzVHvGjWOuF1Ljc44b+NXAOazodHkGUHp4I3zCkQ4/sNVJm56h1S278LMR0bY5xBhztGziSbli8zObIw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/parser": "^7.29.7",
|
||||
"@babel/types": "^7.29.7",
|
||||
"@babel/parser": "^7.29.13",
|
||||
"@babel/types": "^7.29.11",
|
||||
"@jridgewell/gen-mapping": "^0.3.12",
|
||||
"@jridgewell/trace-mapping": "^0.3.28",
|
||||
"jsesc": "^3.0.2"
|
||||
@@ -955,9 +955,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-compilation-targets": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz",
|
||||
"integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.13.tgz",
|
||||
"integrity": "sha512-8WxJ4/Vmes3ghacj8VOJy3NW8eyHiFY/4thU/LSBuBE470+igjR/j1ZTNn8fV5VdNrLx6iXUwig526UdOvfP0w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -982,22 +982,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-compilation-targets/node_modules/semver": {
|
||||
"version": "6.3.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
|
||||
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
|
||||
"version": "6.3.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.2.tgz",
|
||||
"integrity": "sha512-GYrCADwpu6yV5OAvchkQ27Z5wwMxWAEMg5NNWjyHh/5DboFFqbdbvYNIvZouQpxXyeMPoTOh0vdPYKlf0ZswdQ==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-compilation-targets/node_modules/yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/@babel/helper-globals": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz",
|
||||
@@ -1023,15 +1016,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-module-transforms": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz",
|
||||
"integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.13.tgz",
|
||||
"integrity": "sha512-/U96wOz+SCoZ7Da/8kjDGltcNY280JPQcRNPK0Hk3PTbdpaiBDQImtC6JsbVwb1HGMr3usZQU2WsWVjhYNmRzw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/helper-module-imports": "^7.29.7",
|
||||
"@babel/helper-validator-identifier": "^7.29.7",
|
||||
"@babel/traverse": "^7.29.7"
|
||||
"@babel/traverse": "^7.29.13"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
@@ -1051,9 +1044,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-string-parser": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz",
|
||||
"integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.13.tgz",
|
||||
"integrity": "sha512-fWegr+WsEsiMVW4KUNdIIzzMTZSzNiqhy0//6+2MRi2d5NFNsL/AYwBQW8eBnRbTBVRnDVzjaJEUqKaCWjMMdQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -1081,27 +1074,27 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helpers": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz",
|
||||
"integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.13.tgz",
|
||||
"integrity": "sha512-laoRpD/kO/iA4CDlgJbixHekg7oPvWALK0PQyK3Ob2I2njZRvVAD77RfWCIImtqWCq6KijoBoZK7bp/HlTWASQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/template": "^7.29.7",
|
||||
"@babel/types": "^7.29.7"
|
||||
"@babel/types": "^7.29.11"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/parser": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz",
|
||||
"integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.13.tgz",
|
||||
"integrity": "sha512-fJM50YYjlTiBhgdkGmC1tBpv+FMAyo4annf2X6Aks+0yRlaiVkYW/3f07xAfVVkYIoZ9fFZp+mEThUFnuybhkQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/types": "^7.29.7"
|
||||
"@babel/types": "^7.29.11"
|
||||
},
|
||||
"bin": {
|
||||
"parser": "bin/babel-parser.js"
|
||||
@@ -1135,18 +1128,18 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/traverse": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz",
|
||||
"integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==",
|
||||
"version": "7.29.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.13.tgz",
|
||||
"integrity": "sha512-0ve0xMlFA73dgY1oyJRY6RoW+/bncdcOQqg6jjYcVfXBK90OdLXjm9R6lmRetdZVrPjXkKIoMVBqYFPjcAlZ4Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.29.7",
|
||||
"@babel/generator": "^7.29.7",
|
||||
"@babel/generator": "^7.29.13",
|
||||
"@babel/helper-globals": "^7.29.7",
|
||||
"@babel/parser": "^7.29.7",
|
||||
"@babel/parser": "^7.29.13",
|
||||
"@babel/template": "^7.29.7",
|
||||
"@babel/types": "^7.29.7",
|
||||
"@babel/types": "^7.29.11",
|
||||
"debug": "^4.3.1"
|
||||
},
|
||||
"engines": {
|
||||
@@ -1154,13 +1147,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/types": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz",
|
||||
"integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==",
|
||||
"version": "7.29.11",
|
||||
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.11.tgz",
|
||||
"integrity": "sha512-WCsemW40Cpc6gouC1SrY6PBaZ+oLRlEVY9XBvZBI7il/HyUrb2nJ505wE1MHEFyDZNnrWDEuy4jL50WVHfTTSQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/helper-string-parser": "^7.29.7",
|
||||
"@babel/helper-string-parser": "^7.29.11",
|
||||
"@babel/helper-validator-identifier": "^7.29.7"
|
||||
},
|
||||
"engines": {
|
||||
@@ -7460,18 +7453,18 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/body-parser": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz",
|
||||
"integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==",
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.4.0.tgz",
|
||||
"integrity": "sha512-Ifk1KcLrxaGt1occQagQNFNNOqRt1LFuehEnHHzB8Z6cRgIxRU1TBaiIsUiCdA+gy5PL7Y1IX31DNKkRmxtXjQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bytes": "^3.1.2",
|
||||
"content-type": "^2.0.0",
|
||||
"content-type": "^2.1.0",
|
||||
"debug": "^4.4.3",
|
||||
"http-errors": "^2.0.1",
|
||||
"iconv-lite": "^0.7.2",
|
||||
"iconv-lite": "^0.7.3",
|
||||
"on-finished": "^2.4.1",
|
||||
"qs": "^6.15.2",
|
||||
"qs": "^6.16.1",
|
||||
"raw-body": "^3.0.2",
|
||||
"type-is": "^2.1.0"
|
||||
},
|
||||
@@ -7484,9 +7477,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/body-parser/node_modules/content-type": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz",
|
||||
"integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==",
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
|
||||
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
@@ -9581,9 +9574,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/fastify": {
|
||||
"version": "5.12.5",
|
||||
"resolved": "https://registry.npmjs.org/fastify/-/fastify-5.12.5.tgz",
|
||||
"integrity": "sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==",
|
||||
"version": "5.12.7",
|
||||
"resolved": "https://registry.npmjs.org/fastify/-/fastify-5.12.7.tgz",
|
||||
"integrity": "sha512-v4pEPsTRPuqlDIYiQFB+BHjVN4SdHX9u0ABk0ZDUKbbdY1vsNryUd1j4Z+jQEZajvYIVk047Kimva9/AJ79GYw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -16587,9 +16580,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.30.1",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz",
|
||||
"integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==",
|
||||
"version": "7.30.2",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.30.2.tgz",
|
||||
"integrity": "sha512-mLyPQnHwUxaM/rideoK/+KL8DaAJLRl4aBCfmX444aCSLnIaomqPLtNRe2jdCYcsk5wZzXrDIQC8ZeYNRAYyIA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
@@ -18141,6 +18134,13 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/yaml": {
|
||||
"version": "2.9.0",
|
||||
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
|
||||
@@ -18233,9 +18233,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz",
|
||||
"integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==",
|
||||
"version": "4.6.7",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz",
|
||||
"integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
@@ -18300,7 +18300,7 @@
|
||||
"mime-types": "^2.1.35",
|
||||
"murmurhash": "^2.0.1",
|
||||
"mysql2": "^3.24.6",
|
||||
"nodemailer": "^10.1.4",
|
||||
"nodemailer": "^10.1.7",
|
||||
"openai": "^6.34.0",
|
||||
"otpauth": "^9.5.2",
|
||||
"pg": "^8.21.0",
|
||||
@@ -18312,10 +18312,10 @@
|
||||
"svg-captcha": "^1.4.0",
|
||||
"together-ai": "^0.33.0",
|
||||
"ua-parser-js": "^1.0.41",
|
||||
"undici": "^7.30.1",
|
||||
"undici": "^7.30.2",
|
||||
"uuid": "^14.0.3",
|
||||
"validator": "^13.15.35",
|
||||
"ws": "^8.22.0"
|
||||
"ws": "^8.22.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bcrypt": "^6.0.0",
|
||||
@@ -18343,10 +18343,19 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"src/backend/node_modules/nodemailer": {
|
||||
"version": "10.1.7",
|
||||
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.7.tgz",
|
||||
"integrity": "sha512-OQa6FDln5fLttmqSbjDiRYvFDOFNrBqfiOwHPlEbURil3b+vPB1v8xpi+Sp46drzNDlsTnNsS502vthmz56r4w==",
|
||||
"license": "MIT-0",
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"src/backend/node_modules/ws": {
|
||||
"version": "8.22.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz",
|
||||
"integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==",
|
||||
"version": "8.22.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.22.2.tgz",
|
||||
"integrity": "sha512-/rEZ2ejown2N5gxVTHUu9bVVC7ttu0/Tgzf5hbTOixR3Fs7mbbXgn2DbdbcKgJOmpQMtrIUWivtYuVMnp/W/yA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
|
||||
@@ -125,7 +125,6 @@ describe('core without a teams registration', () => {
|
||||
actor: { user: row },
|
||||
} as never,
|
||||
{ title: 'Puter' } as never,
|
||||
{} as never,
|
||||
);
|
||||
const params = JSON.parse(
|
||||
/gui\((\{.*?\})\);/s.exec(html)![1].replaceAll('\\u003c', '<'),
|
||||
|
||||
@@ -167,6 +167,38 @@ describe('AbstractDatabaseClient — replica-aware reads', () => {
|
||||
expect(client.reads).toEqual(['SELECT 1']);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
describe('tryHardReadAll', () => {
|
||||
const reads = [{ query: 'SELECT 1' }, { query: 'SELECT 2' }];
|
||||
|
||||
it('keeps replica rows without reading the primary', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
expect(await client.tryHardReadAll(reads)).toEqual([
|
||||
[{ id: 1 }],
|
||||
[{ id: 1 }],
|
||||
]);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([[], new Error('replica unavailable')])(
|
||||
'reads every query on the primary when the replica has none',
|
||||
async (replica) => {
|
||||
const client = new FixtureDatabaseClient(replica, [{ id: 2 }]);
|
||||
expect(await client.tryHardReadAll(reads)).toEqual([
|
||||
[{ id: 2 }],
|
||||
[{ id: 2 }],
|
||||
]);
|
||||
expect(client.preads).toEqual(['SELECT 1', 'SELECT 2']);
|
||||
},
|
||||
);
|
||||
|
||||
it('issues one read per query when there is no replica', async () => {
|
||||
const client = new FixtureDatabaseClient([], [{ id: 2 }], false);
|
||||
expect(await client.tryHardReadAll(reads)).toEqual([[], []]);
|
||||
expect(client.reads).toEqual(['SELECT 1', 'SELECT 2']);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — SQL generation', () => {
|
||||
|
||||
@@ -172,6 +172,34 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
return result.value;
|
||||
}
|
||||
|
||||
/** Reads the primary only when every replica read is empty or fails. */
|
||||
@Span('db.tryHardReadAll', (reads: unknown[]) => ({
|
||||
'db.batch_size': reads.length,
|
||||
}))
|
||||
async tryHardReadAll(
|
||||
reads: { query: string; params?: unknown[] }[],
|
||||
): Promise<Record<string, unknown>[][]> {
|
||||
if (!this.hasReadReplica()) {
|
||||
return Promise.all(
|
||||
reads.map(({ query, params = [] }) => this.read(query, params)),
|
||||
);
|
||||
}
|
||||
const onReplica = await Promise.all(
|
||||
reads.map(async ({ query, params = [] }) => {
|
||||
try {
|
||||
return await this.replicaRead(query, params);
|
||||
} catch {
|
||||
// A failed replica read counts as a miss.
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (onReplica.some((rows) => rows.length > 0)) return onReplica;
|
||||
return Promise.all(
|
||||
reads.map(({ query, params = [] }) => this.pread(query, params)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the `.sql` files named `<engineName>*` in each configured migration
|
||||
* directory, in filename order. Files must be idempotent: nothing records
|
||||
|
||||
@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
|
||||
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
|
||||
|
||||
/** Highest schema version the migration table can reach. */
|
||||
const CURRENT_SCHEMA_VERSION = 98;
|
||||
const CURRENT_SCHEMA_VERSION = 99;
|
||||
|
||||
/**
|
||||
* These suites migrate real files on disk. Idle they finish in well under a
|
||||
@@ -705,6 +705,60 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
|
||||
}
|
||||
});
|
||||
|
||||
it('fills a share row missing its entry owner from 0103, invites left alone', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-share-owner-'));
|
||||
const path = join(dir, 'puter.sqlite');
|
||||
try {
|
||||
const before = new SqliteDatabaseClient(
|
||||
sqliteConfig({ inMemory: false, path }),
|
||||
);
|
||||
await before.onServerStart();
|
||||
const [user] = (await before.read(
|
||||
'SELECT MIN(`id`) AS id FROM `user`',
|
||||
)) as { id: number }[];
|
||||
await before.write(
|
||||
'INSERT INTO `fsentries` (`uuid`, `name`, `user_id`, `modified`) ' +
|
||||
'VALUES (?, ?, ?, ?)',
|
||||
['22222222-2222-4222-8222-222222222222', 'owned.txt', user.id, 0],
|
||||
);
|
||||
const [entry] = (await before.read(
|
||||
'SELECT `id` FROM `fsentries` WHERE `uuid` = ?',
|
||||
['22222222-2222-4222-8222-222222222222'],
|
||||
)) as { id: number }[];
|
||||
await before.write(
|
||||
'INSERT INTO `share` (`uid`, `issuer_user_id`, ' +
|
||||
'`recipient_email`, `fsentry_id`) VALUES (?, ?, ?, ?)',
|
||||
['share-no-owner', user.id, '[email protected]', entry.id],
|
||||
);
|
||||
await before.write(
|
||||
'INSERT INTO `share` (`uid`, `issuer_user_id`, ' +
|
||||
'`recipient_email`) VALUES (?, ?, ?)',
|
||||
['share-invite', user.id, '[email protected]'],
|
||||
);
|
||||
// The full schema as it stood before 0103, so only 0103 reruns.
|
||||
await before.write('PRAGMA user_version = 98');
|
||||
before.onServerShutdown();
|
||||
|
||||
const after = new SqliteDatabaseClient(
|
||||
sqliteConfig({ inMemory: false, path }),
|
||||
);
|
||||
await after.onServerStart();
|
||||
await expect(
|
||||
after.read(
|
||||
'SELECT `uid`, `entry_owner_user_id` FROM `share` ' +
|
||||
'WHERE `uid` IN (?, ?) ORDER BY `uid`',
|
||||
['share-invite', 'share-no-owner'],
|
||||
),
|
||||
).resolves.toEqual([
|
||||
{ uid: 'share-invite', entry_owner_user_id: null },
|
||||
{ uid: 'share-no-owner', entry_owner_user_id: user.id },
|
||||
]);
|
||||
after.onServerShutdown();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('indexes notification.created_at for the retention sweep', async () => {
|
||||
const rows = await client.read('PRAGMA index_list(`notification`)');
|
||||
expect(rows.map((r) => String(r.name))).toContain(
|
||||
|
||||
@@ -137,6 +137,7 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
|
||||
[95, ['0100_notification-send-key.sql']],
|
||||
[96, ['0101_user-scoped-handlers.sql']],
|
||||
[97, ['0102_notification-scope-id-index.sql']],
|
||||
[98, ['0103_share-entry-owner-backfill.sql']],
|
||||
];
|
||||
|
||||
export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Fills `share`.`entry_owner_user_id` on any row that names an entry but was
|
||||
-- written without the owner. Mirrors SQLite migration 0103. No per-file
|
||||
-- applied-state tracking; a replay finds nothing to fill.
|
||||
|
||||
UPDATE `share` JOIN `fsentries` ON `fsentries`.`id` = `share`.`fsentry_id`
|
||||
SET `share`.`entry_owner_user_id` = `fsentries`.`user_id`
|
||||
WHERE `share`.`fsentry_id` IS NOT NULL
|
||||
AND `share`.`entry_owner_user_id` IS NULL;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Fills "share".entry_owner_user_id on any row that names an entry but was
|
||||
-- written without the owner. Mirrors SQLite migration 0103. No per-file
|
||||
-- applied-state tracking; a replay finds nothing to fill.
|
||||
|
||||
UPDATE "share"
|
||||
SET entry_owner_user_id = f.user_id
|
||||
FROM fsentries f
|
||||
WHERE f.id = "share".fsentry_id
|
||||
AND "share".fsentry_id IS NOT NULL
|
||||
AND "share".entry_owner_user_id IS NULL;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Fills `share`.`entry_owner_user_id` on any row that names an entry but was
|
||||
-- written without the owner, so reads can rely on the column alone. Rows that
|
||||
-- name no entry (invites) keep a NULL owner. A replay finds nothing to fill.
|
||||
|
||||
UPDATE `share`
|
||||
SET `entry_owner_user_id` = (
|
||||
SELECT `user_id` FROM `fsentries` WHERE `fsentries`.`id` = `share`.`fsentry_id`
|
||||
)
|
||||
WHERE `fsentry_id` IS NOT NULL
|
||||
AND `entry_owner_user_id` IS NULL;
|
||||
@@ -168,6 +168,204 @@ describe('EmailClient — sending', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('EmailClient — streams', () => {
|
||||
const STREAMS = {
|
||||
notify: {
|
||||
from: '"Notices" <[email protected]>',
|
||||
headers: { 'X-Example-Stream': 'notify', 'X-Example-Kind': 'n' },
|
||||
tagsHeader: 'X-Example-Tags',
|
||||
},
|
||||
plain: { headers: { 'X-Example-Stream': 'plain' } },
|
||||
};
|
||||
const streamClient = () =>
|
||||
startClient({
|
||||
email: { jsonTransport: true, from: FROM, streams: STREAMS },
|
||||
} as unknown as Partial<IConfig>);
|
||||
const base = { to: '[email protected]', subject: 's', text: 't' };
|
||||
|
||||
it('adds the stream headers and from address to the message', async () => {
|
||||
const info = await streamClient().sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
});
|
||||
|
||||
const message = sentMessage(info);
|
||||
expect(message.headers).toMatchObject({
|
||||
'X-Example-Stream': 'notify',
|
||||
'X-Example-Kind': 'n',
|
||||
});
|
||||
expect(message.from.address).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('lets caller headers and from win over the stream', async () => {
|
||||
const info = await streamClient().sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
from: '[email protected]',
|
||||
headers: { 'x-example-stream': 'mine' },
|
||||
});
|
||||
|
||||
const message = sentMessage(info);
|
||||
expect(message.from.address).toBe('[email protected]');
|
||||
expect(message.headers?.['x-example-stream']).toBe('mine');
|
||||
expect(message.headers).not.toHaveProperty('X-Example-Stream');
|
||||
expect(message.headers?.['X-Example-Kind']).toBe('n');
|
||||
});
|
||||
|
||||
it('keeps the default from address for a stream without one', async () => {
|
||||
const info = await streamClient().sendRaw({ ...base, stream: 'plain' });
|
||||
expect(sentMessage(info).from.address).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('leaves the message unchanged for an unknown or missing stream', async () => {
|
||||
const client = streamClient();
|
||||
for (const stream of ['nope', undefined]) {
|
||||
const message = sentMessage(
|
||||
await client.sendRaw({ ...base, stream, tags: { a: 'b' } }),
|
||||
);
|
||||
expect(message.from.address).toBe('[email protected]');
|
||||
expect(message.headers ?? {}).toEqual({});
|
||||
}
|
||||
});
|
||||
|
||||
it('renders valid tags into the stream tags header and drops the rest', async () => {
|
||||
const info = await streamClient().sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
tags: {
|
||||
kind: 'share',
|
||||
'app.id': 'a-1:x',
|
||||
'bad key': 'v',
|
||||
ok: 'bad value',
|
||||
long: 'x'.repeat(129),
|
||||
},
|
||||
});
|
||||
expect(sentMessage(info).headers?.['X-Example-Tags']).toBe(
|
||||
'kind=share, app.id=a-1:x',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps a tags header the caller set', async () => {
|
||||
const info = await streamClient().sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
headers: { 'X-Example-Tags': 'own=1' },
|
||||
tags: { kind: 'share' },
|
||||
});
|
||||
expect(sentMessage(info).headers?.['X-Example-Tags']).toBe('own=1');
|
||||
});
|
||||
|
||||
it('sends no tags header when the stream names none or no tag survives', async () => {
|
||||
const client = streamClient();
|
||||
const plain = sentMessage(
|
||||
await client.sendRaw({
|
||||
...base,
|
||||
stream: 'plain',
|
||||
tags: { kind: 'share' },
|
||||
}),
|
||||
);
|
||||
expect(plain.headers).toEqual({ 'X-Example-Stream': 'plain' });
|
||||
|
||||
const dropped = sentMessage(
|
||||
await client.sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
tags: { 'bad key': 'v' },
|
||||
}),
|
||||
);
|
||||
expect(dropped.headers).not.toHaveProperty('X-Example-Tags');
|
||||
});
|
||||
|
||||
it('does not hand stream or tags to the transport', async () => {
|
||||
const info = await streamClient().sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
tags: { kind: 'share' },
|
||||
});
|
||||
const raw = JSON.parse((info as { message: string }).message);
|
||||
expect(raw).not.toHaveProperty('stream');
|
||||
expect(raw).not.toHaveProperty('tags');
|
||||
});
|
||||
|
||||
it('keeps streams out of the transport options', async () => {
|
||||
const nodemailer = (await import('nodemailer')).default;
|
||||
const create = vi.spyOn(nodemailer, 'createTransport');
|
||||
try {
|
||||
streamClient();
|
||||
expect(create).toHaveBeenCalledTimes(1);
|
||||
const [options] = create.mock.calls[0] as [Record<string, unknown>];
|
||||
expect(options).not.toHaveProperty('streams');
|
||||
expect(options).not.toHaveProperty('from');
|
||||
expect(options).toMatchObject({ jsonTransport: true });
|
||||
} finally {
|
||||
create.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('forwards the template send options to sendRaw', async () => {
|
||||
const client = streamClient();
|
||||
const sendRaw = vi.spyOn(client, 'sendRaw').mockResolvedValue(null);
|
||||
|
||||
await client.send(
|
||||
'[email protected]',
|
||||
'email_verification_code',
|
||||
{ code: '1' },
|
||||
{ stream: 'notify', replyTo: '[email protected]' },
|
||||
);
|
||||
|
||||
expect(sendRaw).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
stream: 'notify',
|
||||
replyTo: '[email protected]',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('applies the stream from address to a template send', async () => {
|
||||
const info = await streamClient().send(
|
||||
'[email protected]',
|
||||
'email_verification_code',
|
||||
{ code: '1' },
|
||||
{ stream: 'notify' },
|
||||
);
|
||||
expect(sentMessage(info).from.address).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('reads a stream only from its own config entry', async () => {
|
||||
const inherited = Object.create({
|
||||
base: { from: '[email protected]', headers: { 'X-Example-B': '1' } },
|
||||
});
|
||||
const client = startClient({
|
||||
email: { jsonTransport: true, from: FROM, streams: inherited },
|
||||
} as unknown as Partial<IConfig>);
|
||||
const message = sentMessage(
|
||||
await client.sendRaw({ ...base, stream: 'base' }),
|
||||
);
|
||||
expect(message.from.address).toBe('[email protected]');
|
||||
expect(message.headers ?? {}).toEqual({});
|
||||
});
|
||||
|
||||
it('prints the stream name and no tags when no transport is configured', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const client = new EmailClient({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
} as unknown as IConfig);
|
||||
await client.sendRaw({
|
||||
...base,
|
||||
stream: 'notify',
|
||||
tags: { kind: 'share' },
|
||||
});
|
||||
const printed = warn.mock.calls[0][1] as Record<string, unknown>;
|
||||
expect(printed).not.toHaveProperty('tags');
|
||||
expect(printed.stream).toBe('notify');
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('EmailClient — template rendering', () => {
|
||||
it('renders the code into both subject and body', async () => {
|
||||
const client = startClient();
|
||||
|
||||
@@ -69,14 +69,50 @@ export interface SendMailOptions {
|
||||
attachments?: EmailAttachment[];
|
||||
/** Extra message headers (e.g. List-Unsubscribe), passed to the transport. */
|
||||
headers?: Record<string, string>;
|
||||
/** Names the `email.streams` entry whose from address and headers apply. */
|
||||
stream?: string;
|
||||
/** Rendered into the stream's `tagsHeader`; ignored when it has none. */
|
||||
tags?: Record<string, string>;
|
||||
}
|
||||
|
||||
/** Options a templated {@link EmailClient.send} forwards to `sendRaw`. */
|
||||
export type SendTemplateOptions = Pick<SendMailOptions, 'replyTo' | 'stream'>;
|
||||
|
||||
const TAG_PART = /^[A-Za-z0-9_.:-]{1,128}$/;
|
||||
|
||||
interface CompiledTemplate {
|
||||
subject: ReturnType<typeof template>;
|
||||
html: ReturnType<typeof template>;
|
||||
text?: ReturnType<typeof template>;
|
||||
}
|
||||
|
||||
/** Pairs whose key or value falls outside `TAG_PART` are dropped. */
|
||||
const renderTags = (tags: Record<string, string> | undefined): string =>
|
||||
Object.entries(tags ?? {})
|
||||
.filter(([name, value]) => TAG_PART.test(name) && TAG_PART.test(value))
|
||||
.map(([name, value]) => `${name}=${value}`)
|
||||
.join(', ');
|
||||
|
||||
const hasHeader = (
|
||||
headers: Record<string, string> | undefined,
|
||||
name: string,
|
||||
): boolean =>
|
||||
Object.keys(headers ?? {}).some(
|
||||
(key) => key.toLowerCase() === name.toLowerCase(),
|
||||
);
|
||||
|
||||
// Header names are case-insensitive: `x-a` from `over` replaces `X-A` in `base`.
|
||||
const mergeHeaders = (
|
||||
base: Record<string, string> | undefined,
|
||||
over: Record<string, string> | undefined,
|
||||
): Record<string, string> => {
|
||||
const merged: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(base ?? {})) {
|
||||
if (!hasHeader(over, key)) merged[key] = value;
|
||||
}
|
||||
return { ...merged, ...over };
|
||||
};
|
||||
|
||||
// -- EmailClient ------------------------------------------------------
|
||||
|
||||
/**
|
||||
@@ -111,7 +147,8 @@ export class EmailClient extends PuterClient {
|
||||
return;
|
||||
}
|
||||
|
||||
this.transport = nodemailer.createTransport(emailConf);
|
||||
const { streams: _streams, from: _from, ...transportConf } = emailConf;
|
||||
this.transport = nodemailer.createTransport(transportConf);
|
||||
console.log('[email] transport configured');
|
||||
}
|
||||
|
||||
@@ -125,7 +162,8 @@ export class EmailClient extends PuterClient {
|
||||
/**
|
||||
* Render a template and send it to `to`. `options.replyTo` sets the
|
||||
* Reply-To header (e.g. so a recipient can respond to the originator of the
|
||||
* message rather than the no-reply From address).
|
||||
* message rather than the no-reply From address); `options.stream` is
|
||||
* passed to {@link sendRaw}.
|
||||
*
|
||||
* Returns the transport's send result, or `null` when none is configured —
|
||||
* {@link sendRaw} drops the message instead of throwing. A caller that
|
||||
@@ -135,7 +173,7 @@ export class EmailClient extends PuterClient {
|
||||
to: string,
|
||||
template: T,
|
||||
values: Record<string, unknown> = {},
|
||||
options: { replyTo?: string } = {},
|
||||
options: SendTemplateOptions = {},
|
||||
) {
|
||||
const compiled = this.compiledTemplates[template];
|
||||
if (!compiled) {
|
||||
@@ -143,12 +181,12 @@ export class EmailClient extends PuterClient {
|
||||
}
|
||||
|
||||
return await this.sendRaw({
|
||||
from: this.defaultFrom(),
|
||||
to,
|
||||
subject: compiled.subject(values),
|
||||
html: compiled.html(values),
|
||||
...(compiled.text ? { text: compiled.text(values) } : {}),
|
||||
...(options.replyTo ? { replyTo: options.replyTo } : {}),
|
||||
...(options.stream ? { stream: options.stream } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -159,19 +197,38 @@ export class EmailClient extends PuterClient {
|
||||
* Returns the transport's send result, or `null` when no transport is
|
||||
* configured (the send is a no-op in that case — callers that must not
|
||||
* silently drop mail should check `config.email` before composing, or the
|
||||
* `null` return after).
|
||||
* `null` return after). A configured `stream` supplies the from address
|
||||
* when the caller gives none, and headers the caller's own override.
|
||||
*/
|
||||
async sendRaw(options: SendMailOptions) {
|
||||
const { stream: streamName, tags, ...message } = options;
|
||||
if (!this.transport) {
|
||||
console.warn(
|
||||
'[email] attempted to send email without transport. If you need to send email, configure an SMTP transport in your config file (see docs for details). Email content:',
|
||||
options,
|
||||
{ ...message, ...(streamName ? { stream: streamName } : {}) },
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const streams = this.config.email?.streams;
|
||||
const stream =
|
||||
streamName && streams && Object.hasOwn(streams, streamName)
|
||||
? streams[streamName]
|
||||
: undefined;
|
||||
let headers = mergeHeaders(stream?.headers, message.headers);
|
||||
const tagsValue = stream?.tagsHeader ? renderTags(tags) : '';
|
||||
if (
|
||||
stream?.tagsHeader &&
|
||||
tagsValue &&
|
||||
!hasHeader(message.headers, stream.tagsHeader)
|
||||
) {
|
||||
headers = mergeHeaders(headers, {
|
||||
[stream.tagsHeader]: tagsValue,
|
||||
});
|
||||
}
|
||||
return await this.transport.sendMail({
|
||||
...options,
|
||||
from: options.from ?? this.defaultFrom(),
|
||||
...message,
|
||||
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
||||
from: message.from ?? stream?.from ?? this.defaultFrom(),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ import type {
|
||||
Response as ExpressResponse,
|
||||
} from 'express';
|
||||
import { Actor } from '../../core';
|
||||
import type { EmailTemplateName } from '../email/templates';
|
||||
import type { UsageInput } from '../../services/metering/types';
|
||||
import { FSEntry } from '../../stores/fs/FSEntry';
|
||||
|
||||
@@ -247,6 +248,12 @@ export type EventMap = {
|
||||
requires_phone_verification?: boolean;
|
||||
/** Set by the abuse harness — require card verification post-signup. */
|
||||
requires_card_verification?: boolean;
|
||||
/**
|
||||
* The email stream the confirmation message is sent on: lowercase
|
||||
* letters, digits, `:`, `_` and `-`, up to 64 characters, starting with
|
||||
* a letter or digit. Any other value is ignored.
|
||||
*/
|
||||
email_stream?: string | null;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
'puter.signup.success': {
|
||||
@@ -298,6 +305,8 @@ export type EventMap = {
|
||||
message: string | null;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** Before notification emails; a listener clears `allow` per recipient. */
|
||||
'email.notify.validate': EmailNotifyValidateEvent;
|
||||
// Inbound mail to a feedback address (`abuse@`, `fbl@`, ...), emitted via
|
||||
// `emitAndWait`. A listener that takes it sets `handled` and it reaches no
|
||||
// mailbox; otherwise it is routed like any other message.
|
||||
@@ -736,6 +745,8 @@ export type EventMap = {
|
||||
'outer.permission.flatInvalidated': {
|
||||
entries: Array<{ holderUserId: number; permission: string }>;
|
||||
};
|
||||
/** Links on these nodes were removed; read them from the primary. */
|
||||
'outer.share.linksRevoked': { fsentryIds: number[]; uuids: string[] };
|
||||
/**
|
||||
* A user's subscription set changed, so every process must drop its cached
|
||||
* "does this user have any subscriptions" answer. Carries the counter the
|
||||
@@ -874,8 +885,11 @@ export type EventMap = {
|
||||
session_uids: string[];
|
||||
};
|
||||
|
||||
/** One access token was revoked; only its own connections should drop. */
|
||||
'auth.access-token.revoked': { token_uid: string };
|
||||
/**
|
||||
* One access token was revoked; only its own connections should drop.
|
||||
* `outer.*` for the same reason as `outer.auth.sessions.revoked`.
|
||||
*/
|
||||
'outer.auth.access-token.revoked': { token_uid: string };
|
||||
|
||||
/** Same-cluster eviction for raw realtime WebSockets. */
|
||||
'pubsub.realtime.auth-revoked': {
|
||||
@@ -974,6 +988,21 @@ export type EventMap = {
|
||||
'outer.pubsub.metering.credits-changed': { userUuid: string };
|
||||
} & IExtensionEventMap;
|
||||
|
||||
/** A union of `Names`, each of which has to be an email template. */
|
||||
type TemplateNames<Names extends EmailTemplateName> = Names;
|
||||
|
||||
/** The emails that ask `email.notify.validate`, by template name. */
|
||||
export type EmailNotifyKind = TemplateNames<
|
||||
'file_shared_with_you' | 'file_shared_invite' | 'app-user-feedback'
|
||||
>;
|
||||
|
||||
/** Payload for `email.notify.validate`: one kind of mail, all its recipients. */
|
||||
export type EmailNotifyValidateEvent = {
|
||||
kind: EmailNotifyKind;
|
||||
/** `to` is the address as it will be sent; `allow` starts true. */
|
||||
recipients: Array<{ to: string; allow: boolean }>;
|
||||
};
|
||||
|
||||
/** Payload for `subscription.gate.<surface>` events. */
|
||||
export type SubscriptionGateEvent = {
|
||||
actor: Actor;
|
||||
@@ -1083,11 +1112,10 @@ export type EventKey = keyof EventMap & string;
|
||||
// Generates a wildcard for every non-final dot-separated prefix of K.
|
||||
export type WildcardPrefixes<K extends string> =
|
||||
K extends `${infer Head}.${infer Tail}`
|
||||
?
|
||||
| `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
? | `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
: never;
|
||||
|
||||
export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
|
||||
@@ -1098,10 +1126,7 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
|
||||
|
||||
/** Acked, not acked, no socket anywhere, or sockets only in a peer region. */
|
||||
export type NotifDeliveryOutcome =
|
||||
| 'received'
|
||||
| 'unreceived'
|
||||
| 'no-socket'
|
||||
| 'remote-only';
|
||||
'received' | 'unreceived' | 'no-socket' | 'remote-only';
|
||||
|
||||
/** One socket's receipt for a live notification. */
|
||||
export interface NotifDeliveryResponder {
|
||||
|
||||
@@ -52,8 +52,8 @@ const APP_READ_LIMIT = {
|
||||
/**
|
||||
* Unauthenticated icon serving; no actor to key on, so the bucket is the
|
||||
* address — and an address is a NAT, a campus or a carrier gateway that can
|
||||
* hold hundreds of desktops. Each desktop boot pulls the taskbar's icons at
|
||||
* once, so a single burst from one network is already thousands of requests.
|
||||
* hold hundreds of desktops. Each dashboard load pulls its app icons at once,
|
||||
* so a single burst from one network is already thousands of requests.
|
||||
* Responses are publicly cacheable and usually a redirect, so the ceiling is
|
||||
* not protecting bandwidth; it is there so a client looping on a broken icon
|
||||
* can't spin unbounded.
|
||||
@@ -523,7 +523,7 @@ export class AppController extends PuterController {
|
||||
};
|
||||
|
||||
// Icons are <img src> targets from the GUI (root) AND resolved via
|
||||
// api_base_url in taskbar payloads. Register on both so either origin
|
||||
// api_base_url in app summaries. Register on both so either origin
|
||||
// works without a cross-subdomain redirect.
|
||||
router.get(
|
||||
'/app-icon/:app_uid',
|
||||
|
||||
@@ -1271,7 +1271,7 @@ describe('AppController app-icon rate limit', () => {
|
||||
it('sizes the icon bucket for a whole network rather than one desktop', () => {
|
||||
// Icons are unauthenticated `<img src>` targets, so the only key is
|
||||
// the address — which one NAT shares across every desktop behind it,
|
||||
// and each boot pulls the taskbar's icons in a burst.
|
||||
// and each dashboard load pulls its app icons in a burst.
|
||||
for (const path of ['/app-icon/:app_uid', '/app-icon/:app_uid/:size']) {
|
||||
const route = router.routes.find(
|
||||
(r) => r.method === 'get' && r.path === path,
|
||||
|
||||
@@ -23,8 +23,10 @@ import { makeActor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { createSecret as otpCreateSecret } from '../../services/auth/OTPUtil.js';
|
||||
import { FULL_API_ACCESS } from '../../services/permission/consts.js';
|
||||
import { tokenClaimsDigest } from '../../../gui/src/helpers/guiToken.js';
|
||||
import {
|
||||
createTestUser,
|
||||
lagReadReplica,
|
||||
setupPuterTestEnv,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
@@ -350,6 +352,98 @@ describe('revoke-own-access-token over HTTP', () => {
|
||||
expect(await tokenReadStatus(file.uid, readToken)).not.toBe(200);
|
||||
});
|
||||
|
||||
const ACCESS_TOKEN_QUERIES =
|
||||
/FROM `access_token_permissions`|`access_token_uid` = \?/;
|
||||
const tokenUidOf = (token: string) =>
|
||||
(
|
||||
env.server.services.token.verify('auth', token) as {
|
||||
token_uid: string;
|
||||
}
|
||||
).token_uid;
|
||||
|
||||
it('serves a read URL fetched right after minting while the replica lags', async () => {
|
||||
const owner = env.users.user;
|
||||
const file = await makeFile(owner);
|
||||
const replica = await lagReadReplica(env.server, ACCESS_TOKEN_QUERIES);
|
||||
try {
|
||||
const readToken = await mintReadToken(owner.token, file.uid);
|
||||
expect(await tokenReadStatus(file.uid, readToken)).toBe(200);
|
||||
const cached = await env.server.clients.redis.get(
|
||||
`perms:token:${tokenUidOf(readToken)}`,
|
||||
);
|
||||
expect(cached).toBe(JSON.stringify([`fs:${file.uid}:read`]));
|
||||
} finally {
|
||||
replica.catchUp();
|
||||
}
|
||||
});
|
||||
|
||||
it('mints from a just-created personal API token while the replica lags', async () => {
|
||||
const owner = env.users.user;
|
||||
const file = await makeFile(owner);
|
||||
const replica = await lagReadReplica(env.server, ACCESS_TOKEN_QUERIES);
|
||||
try {
|
||||
const pat = await mintPersonalToken(owner);
|
||||
const readToken = await mintReadToken(pat, file.uid);
|
||||
expect(await tokenReadStatus(file.uid, readToken)).toBe(200);
|
||||
} finally {
|
||||
replica.catchUp();
|
||||
}
|
||||
});
|
||||
|
||||
it('revokes a just-created token by its uuid while the replica lags', async () => {
|
||||
const owner = env.users.user;
|
||||
const file = await makeFile(owner);
|
||||
const { actor } = await env.server.services.auth.authenticate(
|
||||
owner.token,
|
||||
);
|
||||
const replica = await lagReadReplica(env.server, ACCESS_TOKEN_QUERIES);
|
||||
let readToken: string;
|
||||
try {
|
||||
readToken = await mintReadToken(owner.token, file.uid);
|
||||
await env.server.services.auth.revokeAccessToken(
|
||||
actor!,
|
||||
tokenUidOf(readToken),
|
||||
);
|
||||
expect(await tokenReadStatus(file.uid, readToken)).not.toBe(200);
|
||||
} finally {
|
||||
replica.catchUp();
|
||||
}
|
||||
expect(await tokenReadStatus(file.uid, readToken)).not.toBe(200);
|
||||
expect(await whoamiStatus(readToken)).toBe(401);
|
||||
});
|
||||
|
||||
it('keeps a revoked token dead while the replica still shows it live', async ({
|
||||
skip,
|
||||
}) => {
|
||||
// The frozen replica is a sqlite copy.
|
||||
skip(process.env.PUTER_TEST_DB_ENGINE?.toLowerCase() === 'postgres');
|
||||
const owner = env.users.user;
|
||||
const file = await makeFile(owner);
|
||||
const { actor } = await env.server.services.auth.authenticate(
|
||||
owner.token,
|
||||
);
|
||||
const readToken = await mintReadToken(owner.token, file.uid);
|
||||
const tokenUid = tokenUidOf(readToken);
|
||||
const replica = await lagReadReplica(env.server, ACCESS_TOKEN_QUERIES, {
|
||||
mode: 'frozen',
|
||||
});
|
||||
try {
|
||||
await env.server.services.auth.revokeAccessToken(actor!, tokenUid);
|
||||
expect(await tokenReadStatus(file.uid, readToken)).not.toBe(200);
|
||||
expect(
|
||||
await env.server.stores.permission.listAccessTokenPerms(
|
||||
tokenUid,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await env.server.clients.redis.get(`perms:token:${tokenUid}`),
|
||||
).toBeNull();
|
||||
} finally {
|
||||
replica.catchUp();
|
||||
}
|
||||
expect(await whoamiStatus(readToken)).toBe(401);
|
||||
});
|
||||
|
||||
// These three run last in the file's shared `env.users.user`: each one
|
||||
// revokes a session belonging to `owner.token` / `owner.workerToken`, so
|
||||
// later tests can't rely on those credentials still being live.
|
||||
@@ -393,7 +487,10 @@ describe('revoke-own-access-token over HTTP', () => {
|
||||
|
||||
it('a revoked GUI session still gets a reauth_token', async () => {
|
||||
const owner = env.users.user;
|
||||
const decoded = env.server.services.token.verify('auth', owner.token) as {
|
||||
const decoded = env.server.services.token.verify(
|
||||
'auth',
|
||||
owner.token,
|
||||
) as {
|
||||
session_uid: string;
|
||||
};
|
||||
await env.server.stores.session.removeByUuid(decoded.session_uid);
|
||||
@@ -559,7 +656,6 @@ describe('2FA enrollment over HTTP', () => {
|
||||
expect(enabled.status).toBe(200);
|
||||
expect((await otpState(account.userId)).enabled).toBe(true);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -714,4 +810,67 @@ describe('session token helpers over HTTP', () => {
|
||||
'puter_auth_token=',
|
||||
);
|
||||
});
|
||||
|
||||
describe("the shell's guiTokenDigest", () => {
|
||||
/** The session cookie `/session/sync-cookie` sets for `token`. */
|
||||
const sessionCookie = async (token: string) => {
|
||||
const sync = await get('/session/sync-cookie', token);
|
||||
expect(sync.status).toBe(204);
|
||||
const value = /puter_auth_token=([^;]+)/.exec(
|
||||
sync.headers.get('set-cookie') ?? '',
|
||||
)?.[1];
|
||||
expect(value).toBeTruthy();
|
||||
return `puter_auth_token=${value}`;
|
||||
};
|
||||
|
||||
/** What the shell at `path` renders for `guiTokenDigest`. */
|
||||
const shellDigest = async (path: string, cookie?: string) => {
|
||||
const res = await fetch(new URL(path, env.origin), {
|
||||
headers: cookie ? { cookie } : {},
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const html = await res.text();
|
||||
const match = /"guiTokenDigest":(null|"[0-9a-f]{32}")/.exec(html);
|
||||
if (!match) {
|
||||
expect(html).not.toContain('guiTokenDigest');
|
||||
return undefined;
|
||||
}
|
||||
return JSON.parse(match[1]) as string | null;
|
||||
};
|
||||
|
||||
it("names the token /get-gui-token hands over for the session cookie", async () => {
|
||||
const { token } = await signupTemp();
|
||||
const cookie = await sessionCookie(token);
|
||||
|
||||
const gui = await fetch(new URL('/get-gui-token', env.origin), {
|
||||
headers: { cookie },
|
||||
});
|
||||
expect(gui.status).toBe(200);
|
||||
const { token: guiToken } = (await gui.json()) as { token: string };
|
||||
|
||||
const digest = await shellDigest('/', cookie);
|
||||
expect(digest).toBe(await tokenClaimsDigest(guiToken));
|
||||
// The token the browser stored at sign-up is for the same session.
|
||||
expect(await tokenClaimsDigest(token)).toBe(digest);
|
||||
});
|
||||
|
||||
it('is null without a session cookie, even with a token in the URL', async () => {
|
||||
const { token } = await signupTemp();
|
||||
expect(await shellDigest('/')).toBe(null);
|
||||
expect(
|
||||
await shellDigest(`/?auth_token=${encodeURIComponent(token)}`),
|
||||
).toBe(null);
|
||||
});
|
||||
|
||||
it('is left out when the session cookie does not resolve to a session', async () => {
|
||||
const { token } = await signupTemp();
|
||||
const cookie = await sessionCookie(token);
|
||||
await env.server.services.auth.revokeSession(claims(token).session_uid);
|
||||
|
||||
expect(await shellDigest('/', cookie)).toBe(undefined);
|
||||
expect(await shellDigest('/', 'puter_auth_token=garbage')).toBe(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -30,6 +30,7 @@
|
||||
|
||||
import { installSeatStub } from '../../testFixtures/seatStub.js';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { inspect } from 'node:util';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { TOTP } from 'otpauth';
|
||||
import { v4 as uuidv4, v5 as uuidv5 } from 'uuid';
|
||||
@@ -1416,7 +1417,7 @@ describe('AuthController.handleLogin', () => {
|
||||
is_user_token: true,
|
||||
});
|
||||
expect(body.user.directories).toBeTypeOf('object');
|
||||
expect(Array.isArray(body.user.taskbar_items)).toBe(true);
|
||||
expect(body.user).not.toHaveProperty('taskbar_items');
|
||||
});
|
||||
|
||||
// A deleted row still answering from cache reaches the session INSERT.
|
||||
@@ -4765,14 +4766,24 @@ describe('AuthController.handleSendConfirmEmail', () => {
|
||||
).rejects.toMatchObject({ statusCode: 403 });
|
||||
});
|
||||
|
||||
it('rotates the email_confirm_code and returns {} on success', async () => {
|
||||
it('rotates the email_confirm_code and reports the send', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const before = await server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
const send = vi
|
||||
.spyOn(server.clients.email, 'send')
|
||||
.mockResolvedValue({ messageId: 'm' } as never);
|
||||
const res = makeRes();
|
||||
await controller.handleSendConfirmEmail(makeReq({}, { actor }), res);
|
||||
expect(res.body).toEqual({});
|
||||
try {
|
||||
await controller.handleSendConfirmEmail(
|
||||
makeReq({}, { actor }),
|
||||
res,
|
||||
);
|
||||
} finally {
|
||||
send.mockRestore();
|
||||
}
|
||||
expect(res.body).toEqual({ sent: true });
|
||||
const after = await server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
@@ -4782,7 +4793,7 @@ describe('AuthController.handleSendConfirmEmail', () => {
|
||||
|
||||
// The resend is the recovery path for a lost code, so it must not fail
|
||||
// silently either.
|
||||
it('alarms — and still returns {} — when the resend send throws', async () => {
|
||||
it('alarms, and still answers 200 with sent: false, when the resend send throws', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const alarm = vi
|
||||
.spyOn(server.clients.alarm, 'create')
|
||||
@@ -4797,7 +4808,8 @@ describe('AuthController.handleSendConfirmEmail', () => {
|
||||
res,
|
||||
);
|
||||
// Not the caller's fault; still a 200.
|
||||
expect(res.body).toEqual({});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.body).toEqual({ sent: false });
|
||||
|
||||
const raised = alarm.mock.calls.find(
|
||||
(c) => c[0] === 'auth:confirmation-email-send-failed:resend',
|
||||
@@ -7518,6 +7530,46 @@ describe('AuthController.handleSaveAccount', () => {
|
||||
};
|
||||
};
|
||||
|
||||
it('sends the confirmation code on the account confirmation stream', async () => {
|
||||
const { user, actor } = await makeTempActor();
|
||||
await server.stores.user.updateMetadata(user.id, {
|
||||
emailStream: 'review',
|
||||
});
|
||||
const send = vi
|
||||
.spyOn(server.clients.email, 'send')
|
||||
.mockResolvedValue(null);
|
||||
const alarm = vi
|
||||
.spyOn(server.clients.alarm, 'create')
|
||||
.mockImplementation(() => undefined);
|
||||
try {
|
||||
const username = `s_${uniq()}`;
|
||||
await controller.handleSaveAccount(
|
||||
makeReq(
|
||||
{
|
||||
username,
|
||||
email: `${username}@test.local`,
|
||||
password: 'correct-horse-battery',
|
||||
},
|
||||
{ actor },
|
||||
),
|
||||
makeRes(),
|
||||
);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
`${username}@test.local`,
|
||||
'email_verification_code',
|
||||
expect.anything(),
|
||||
{ stream: 'review' },
|
||||
);
|
||||
// The dropped send is reported under its own stage.
|
||||
expect(alarm.mock.calls.map((c) => c[0])).toEqual([
|
||||
'auth:confirmation-email-send-failed:save-account',
|
||||
]);
|
||||
} finally {
|
||||
send.mockRestore();
|
||||
alarm.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects non-temp accounts with 400', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
await expect(
|
||||
@@ -8293,11 +8345,10 @@ describe('AuthController session endpoints', () => {
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('revoke-session: 403 when revoking someone else’s session', async () => {
|
||||
it('revoke-session: 404 when revoking someone else’s session', async () => {
|
||||
const { user: u1 } = await makeUserAndActor();
|
||||
const { actor: a2 } = await makeUserAndActor();
|
||||
// Create a real session for u1 so the lookup succeeds, then attempt
|
||||
// to revoke it as a2 — must 403.
|
||||
// A real session of u1, revoked as a2: answers as a missing one.
|
||||
const sessionRes = await server.services.auth.createSessionToken(
|
||||
u1,
|
||||
{},
|
||||
@@ -8310,7 +8361,7 @@ describe('AuthController session endpoints', () => {
|
||||
),
|
||||
makeRes(),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 403 });
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('rename-session: 400 when uuid param is missing', async () => {
|
||||
@@ -9510,14 +9561,18 @@ describe('AuthController.handleSignup additional branches', () => {
|
||||
},
|
||||
async () => {
|
||||
const username = `cefail_${uniq()}`;
|
||||
const res = makeRes();
|
||||
await controller.handleSignup(
|
||||
makeReq({
|
||||
username,
|
||||
email: `${username}@test.local`,
|
||||
password: 'correct-horse-battery',
|
||||
}),
|
||||
makeRes(),
|
||||
res,
|
||||
);
|
||||
expect(res.body).toMatchObject({
|
||||
confirmation_email_sent: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -9538,6 +9593,93 @@ describe('AuthController.handleSignup additional branches', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('AuthController.handleSignup confirmation email outcome', () => {
|
||||
const signupWithSend = async (send: () => Promise<unknown>) => {
|
||||
const spy = vi
|
||||
.spyOn(server.clients.email, 'send')
|
||||
.mockImplementation(send as never);
|
||||
const alarm = vi
|
||||
.spyOn(server.clients.alarm, 'create')
|
||||
.mockImplementation(() => undefined);
|
||||
const res = makeRes();
|
||||
const username = `ceo_${uniq()}`;
|
||||
let alarms: string[] = [];
|
||||
try {
|
||||
await withSignupValidateOverride(
|
||||
(event) => {
|
||||
event.requires_email_confirmation = true;
|
||||
},
|
||||
() =>
|
||||
controller.handleSignup(
|
||||
makeReq({
|
||||
username,
|
||||
email: `${username}@test.local`,
|
||||
password: 'correct-horse-battery',
|
||||
}),
|
||||
res,
|
||||
),
|
||||
);
|
||||
alarms = alarm.mock.calls.map((c) => String(c[0]));
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
alarm.mockRestore();
|
||||
}
|
||||
return {
|
||||
body: res.body as {
|
||||
proceed: boolean;
|
||||
confirmation_email_sent?: boolean;
|
||||
user: { requires_email_confirmation: boolean };
|
||||
},
|
||||
alarms,
|
||||
};
|
||||
};
|
||||
|
||||
it('reports a send that throws and still completes the signup', async () => {
|
||||
const { body, alarms } = await signupWithSend(async () => {
|
||||
throw new Error('refused');
|
||||
});
|
||||
expect(body.proceed).toBe(true);
|
||||
expect(body.confirmation_email_sent).toBe(false);
|
||||
expect(body.user.requires_email_confirmation).toBeTruthy();
|
||||
expect(alarms).toContain('auth:confirmation-email-send-failed:signup');
|
||||
});
|
||||
|
||||
it('reports a send the transport accepted', async () => {
|
||||
const { body } = await signupWithSend(async () => ({ messageId: 'm' }));
|
||||
expect(body.confirmation_email_sent).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the confirmation stream out of the returned user details', async () => {
|
||||
const res = makeRes();
|
||||
const username = `ces_${uniq()}`;
|
||||
await withSignupValidateOverride(
|
||||
(event) => {
|
||||
(event as Record<string, unknown>).email_stream = 'review';
|
||||
},
|
||||
() =>
|
||||
controller.handleSignup(
|
||||
makeReq({
|
||||
username,
|
||||
email: `${username}@test.local`,
|
||||
password: 'correct-horse-battery',
|
||||
}),
|
||||
res,
|
||||
),
|
||||
);
|
||||
const stored = await server.stores.user.getByUsername(username);
|
||||
expect(stored!.metadata?.emailStream).toBe('review');
|
||||
const details = (res.body as { user: { metadata?: object } }).user;
|
||||
expect(details.metadata ?? {}).not.toHaveProperty('emailStream');
|
||||
});
|
||||
|
||||
it('leaves the field out when no confirmation email is due', async () => {
|
||||
const res = makeRes();
|
||||
await controller.handleSignup(makeReq({ is_temp: true }), res);
|
||||
expect(res.body).toMatchObject({ proceed: true });
|
||||
expect(res.body).not.toHaveProperty('confirmation_email_sent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('AuthController.handleSendPassRecoveryEmail additional branches', () => {
|
||||
it('rejects an invalid email format with 400 (when no username supplied)', async () => {
|
||||
await expect(
|
||||
@@ -10082,6 +10224,77 @@ describe('AuthController.handleRevokeSession additional branches', () => {
|
||||
expect(Array.isArray(body.sessions)).toBe(true);
|
||||
});
|
||||
|
||||
const expiredSessionOf = async (user: { id: number }) => {
|
||||
const row = await server.stores.session.create(user.id, {
|
||||
expires_at: Math.floor(Date.now() / 1000) - 1,
|
||||
});
|
||||
return row.uuid as string;
|
||||
};
|
||||
|
||||
it('revokes an expired session the caller owns', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const uuid = await expiredSessionOf(user);
|
||||
const res = makeRes();
|
||||
await controller.handleRevokeSession(makeReq({ uuid }, { actor }), res);
|
||||
|
||||
const body = res.body as { sessions: Array<{ uuid: string }> };
|
||||
expect(body.sessions.map((s) => s.uuid)).not.toContain(uuid);
|
||||
const rows = await server.clients.db.read(
|
||||
'SELECT `revoked_at` FROM `sessions` WHERE `uuid` = ?',
|
||||
[uuid],
|
||||
);
|
||||
expect(rows[0].revoked_at).not.toBeNull();
|
||||
});
|
||||
|
||||
it('revoking an expired session ends its live child token', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const parent = await expiredSessionOf(user);
|
||||
const child = await server.stores.session.getOrCreateGodmodeToken(
|
||||
user.id,
|
||||
{
|
||||
parentSessionUid: parent,
|
||||
appUid: `app-${uuidv4()}`,
|
||||
expiresAt: Math.floor(Date.now() / 1000) + 3600,
|
||||
maxAgeSeconds: 3600,
|
||||
label: 'Godmode app',
|
||||
auth_id: user.uuid,
|
||||
},
|
||||
);
|
||||
const childUuid = child!.row.uuid as string;
|
||||
expect(await server.stores.session.getByUuid(childUuid)).toBeTruthy();
|
||||
|
||||
await controller.handleRevokeSession(
|
||||
makeReq({ uuid: parent }, { actor }),
|
||||
makeRes(),
|
||||
);
|
||||
|
||||
expect(await server.stores.session.getByUuid(childUuid)).toBeNull();
|
||||
});
|
||||
|
||||
it('answers 404 for an expired session of another user', async () => {
|
||||
const { user: owner } = await makeUserAndActor();
|
||||
const { actor } = await makeUserAndActor();
|
||||
const uuid = await expiredSessionOf(owner);
|
||||
await expect(
|
||||
controller.handleRevokeSession(
|
||||
makeReq({ uuid }, { actor }),
|
||||
makeRes(),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('answers 404 for a session that is already revoked', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const row = await server.stores.session.create(user.id);
|
||||
await server.stores.session.removeByUuid(row.uuid);
|
||||
await expect(
|
||||
controller.handleRevokeSession(
|
||||
makeReq({ uuid: row.uuid }, { actor }),
|
||||
makeRes(),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('refuses to revoke the caller’s OWN current session row (400)', async () => {
|
||||
// Invariant: a self-revoke leaves the client in an ambiguous
|
||||
// identity state because the response can't write fresh auth
|
||||
@@ -10562,10 +10775,10 @@ describe('AuthController.loginWait audience binding', () => {
|
||||
// party holding the session id is somewhere else entirely.
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
|
||||
const { res, outcome } = await waitWithRelay(
|
||||
uuidv4(),
|
||||
{ origin: 'https://evil.test' },
|
||||
token,
|
||||
const session = uuidv4();
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: 'https://evil.test' }],
|
||||
{ during: () => setToken(session, token) },
|
||||
);
|
||||
// Same 408 the empty path returns — a mismatched caller must not be
|
||||
// able to tell "nothing arrived" from "something arrived for someone
|
||||
@@ -10641,10 +10854,10 @@ describe('AuthController.loginWait audience binding', () => {
|
||||
);
|
||||
const guiToken = (loginRes.body as { token: string }).token;
|
||||
|
||||
const { res, outcome } = await waitWithRelay(
|
||||
uuidv4(),
|
||||
{ origin: OPENER },
|
||||
guiToken,
|
||||
const session = uuidv4();
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{ during: () => setToken(session, guiToken) },
|
||||
);
|
||||
expect(outcome).toMatchObject({ statusCode: 408 });
|
||||
expect(res.body).toBeUndefined();
|
||||
@@ -10659,14 +10872,406 @@ describe('AuthController.loginWait audience binding', () => {
|
||||
{ keyid: 'v2' },
|
||||
);
|
||||
|
||||
const { res, outcome } = await waitWithRelay(
|
||||
uuidv4(),
|
||||
{ origin: OPENER },
|
||||
forged,
|
||||
const session = uuidv4();
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{ during: () => setToken(session, forged) },
|
||||
);
|
||||
expect(outcome).toMatchObject({ statusCode: 408 });
|
||||
expect(res.body).toBeUndefined();
|
||||
});
|
||||
|
||||
/** Post a token to `/login/set` with no wait running. */
|
||||
const setToken = (session: string, token: string) =>
|
||||
controller.loginSet(makeReq({ session, auth_token: token }), makeRes());
|
||||
|
||||
/**
|
||||
* Run waits on fake timers so the 10 s window passes without real waiting.
|
||||
* `during` runs once every wait has subscribed; `now` pins the clock.
|
||||
*/
|
||||
const runWaits = async (
|
||||
calls: Array<{ session: string; origin: string; clientGone?: boolean }>,
|
||||
opts: { now?: number; during?: () => Promise<unknown> } = {},
|
||||
) => {
|
||||
const subscribed = vi.spyOn(eventClient, 'on');
|
||||
vi.useFakeTimers({
|
||||
toFake:
|
||||
opts.now === undefined
|
||||
? ['setTimeout']
|
||||
: ['setTimeout', 'Date'],
|
||||
});
|
||||
if (opts.now !== undefined) vi.setSystemTime(opts.now);
|
||||
const tick = () => new Promise((r) => setImmediate(r));
|
||||
try {
|
||||
let pending = calls.length;
|
||||
const runs = calls.map(({ session, origin, clientGone }) => {
|
||||
const res = makeRes();
|
||||
const req = makeReq({ session }, { headers: { origin } });
|
||||
if (clientGone) {
|
||||
Object.assign(req.socket, { destroyed: true });
|
||||
}
|
||||
const outcome = controller
|
||||
.loginWait(req, res)
|
||||
.then(
|
||||
() => 'ok' as const,
|
||||
(e: unknown) => e,
|
||||
)
|
||||
.finally(() => {
|
||||
pending--;
|
||||
});
|
||||
return { res, outcome };
|
||||
});
|
||||
const allSubscribed = () =>
|
||||
calls.every(({ session }) =>
|
||||
subscribed.mock.calls.some(
|
||||
([key]) => key === `pubsub.login.${session}`,
|
||||
),
|
||||
);
|
||||
for (let i = 0; i < 200 && pending > 0 && !allSubscribed(); i++) {
|
||||
await tick();
|
||||
}
|
||||
for (let i = 0; i < 20; i++) await tick();
|
||||
await opts.during?.();
|
||||
for (let i = 0; i < 200 && pending > 0; i++) {
|
||||
await tick();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
}
|
||||
return await Promise.all(
|
||||
runs.map(async ({ res, outcome }) => ({
|
||||
res,
|
||||
outcome: await outcome,
|
||||
})),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
subscribed.mockRestore();
|
||||
}
|
||||
};
|
||||
|
||||
const waitAlone = async (session: string, origin: string, now?: number) =>
|
||||
(await runWaits([{ session, origin }], { now }))[0];
|
||||
|
||||
it('delivers a token set before the wait subscribes', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
await setToken(session, token);
|
||||
const { res, outcome } = await waitAlone(session, OPENER);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it('delivers a token set between one wait timing out and the next', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
const first = await waitAlone(session, OPENER);
|
||||
expect(first.outcome).toMatchObject({ statusCode: 408 });
|
||||
|
||||
await setToken(session, token);
|
||||
const { res, outcome } = await waitAlone(session, OPENER);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it('hands a token out only once', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
await setToken(session, token);
|
||||
expect((await waitAlone(session, OPENER)).outcome).toBe('ok');
|
||||
|
||||
const again = await waitAlone(session, OPENER);
|
||||
expect(again.outcome).toMatchObject({ statusCode: 408 });
|
||||
expect(again.res.body).toBeUndefined();
|
||||
});
|
||||
|
||||
it('withholds a stored token from a mismatched Origin and keeps it for the matching one', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
await setToken(session, token);
|
||||
const other = await waitAlone(session, 'https://other.test');
|
||||
expect(other.outcome).toMatchObject({ statusCode: 408 });
|
||||
expect(other.res.body).toBeUndefined();
|
||||
|
||||
const { res, outcome } = await waitAlone(session, OPENER);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it('drops a stored token after a couple of minutes', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const early = uuidv4();
|
||||
const late = uuidv4();
|
||||
await setToken(early, token);
|
||||
await setToken(late, token);
|
||||
|
||||
const start = Date.now();
|
||||
expect((await waitAlone(early, OPENER, start + 110_000)).outcome).toBe(
|
||||
'ok',
|
||||
);
|
||||
const expired = await waitAlone(late, OPENER, start + 121_000);
|
||||
expect(expired.outcome).toMatchObject({ statusCode: 408 });
|
||||
});
|
||||
|
||||
it('keeps waiting past a stored token for another app and delivers the real one', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const otherToken = await mintAppToken(actor, 'https://other.test');
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
await setToken(session, otherToken);
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{
|
||||
during: () => setToken(session, token),
|
||||
},
|
||||
);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it('still relays to a subscribed wait when storing the token fails', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
const failingSet = vi
|
||||
.spyOn(server.clients.redis, 'set')
|
||||
.mockRejectedValueOnce(new Error('unavailable'));
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
let setRes: MockRes | undefined;
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{
|
||||
during: async () => {
|
||||
setRes = makeRes();
|
||||
await controller.loginSet(
|
||||
makeReq({ session, auth_token: token }),
|
||||
setRes,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(setRes?.body).toEqual({ success: true });
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
} finally {
|
||||
failingSet.mockRestore();
|
||||
warn.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('removes the stored copy once a subscribed wait takes the token', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
const { outcome } = await waitWithRelay(
|
||||
session,
|
||||
{ origin: OPENER },
|
||||
token,
|
||||
);
|
||||
expect(outcome).toBe('ok');
|
||||
|
||||
const later = await waitAlone(session, OPENER);
|
||||
expect(later.outcome).toMatchObject({ statusCode: 408 });
|
||||
});
|
||||
|
||||
it('hands a stored token to only one of two concurrent waits', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
await setToken(session, token);
|
||||
const results = await runWaits([
|
||||
{ session, origin: OPENER },
|
||||
{ session, origin: OPENER },
|
||||
]);
|
||||
const outcomes = results.map((r) => r.outcome);
|
||||
expect(outcomes.filter((o) => o === 'ok')).toHaveLength(1);
|
||||
expect(outcomes.filter((o) => o !== 'ok')).toEqual([
|
||||
expect.objectContaining({ statusCode: 408 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('relays a token to every wait already subscribed', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
const results = await runWaits(
|
||||
[
|
||||
{ session, origin: OPENER },
|
||||
{ session, origin: OPENER },
|
||||
],
|
||||
{ during: () => setToken(session, token) },
|
||||
);
|
||||
expect(results.map((r) => r.outcome)).toEqual(['ok', 'ok']);
|
||||
for (const { res } of results) {
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps the stored copy when the subscribed wait has lost its client', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
await runWaits([{ session, origin: OPENER, clientGone: true }], {
|
||||
during: () => setToken(session, token),
|
||||
});
|
||||
|
||||
const { res, outcome } = await waitAlone(session, OPENER);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it('still relays to a subscribed wait when the stored copy cannot be read or removed', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
const failingGet = vi
|
||||
.spyOn(server.clients.redis, 'get')
|
||||
.mockRejectedValue(new Error('unavailable'));
|
||||
const failingEval = vi
|
||||
.spyOn(server.clients.redis, 'eval')
|
||||
.mockRejectedValue(new Error('unavailable'));
|
||||
try {
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{ during: () => setToken(session, token) },
|
||||
);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
} finally {
|
||||
failingGet.mockRestore();
|
||||
failingEval.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('answers a subscribed wait when a second token is set right after the first', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const first = await mintAppToken(actor, OPENER);
|
||||
// A later `iat` makes the second token differ from the first.
|
||||
await new Promise((r) => setTimeout(r, 1100));
|
||||
const second = await mintAppToken(actor, OPENER);
|
||||
expect(second).not.toBe(first);
|
||||
const session = uuidv4();
|
||||
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{
|
||||
during: async () => {
|
||||
await setToken(session, first);
|
||||
await setToken(session, second);
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(first);
|
||||
});
|
||||
|
||||
it('keeps a subscribed wait listening past a token for another app', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const otherToken = await mintAppToken(actor, 'https://other.test');
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
|
||||
const [{ res, outcome }] = await runWaits(
|
||||
[{ session, origin: OPENER }],
|
||||
{
|
||||
during: async () => {
|
||||
await setToken(session, otherToken);
|
||||
await setToken(session, token);
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it("keeps a stored token when a later set carries another app's token or an unsigned one", async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const otherToken = await mintAppToken(actor, 'https://other.test');
|
||||
const session = uuidv4();
|
||||
|
||||
await setToken(session, token);
|
||||
await setToken(session, otherToken);
|
||||
await setToken(session, 'x'.repeat(100_000));
|
||||
|
||||
const { res, outcome } = await waitAlone(session, OPENER);
|
||||
expect(outcome).toBe('ok');
|
||||
expect((res.body as { auth_token: string }).auth_token).toBe(token);
|
||||
});
|
||||
|
||||
it('stores only a token this server signed for an app', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const real = await mintAppToken(actor, OPENER);
|
||||
const forged = jwt.sign(
|
||||
jwt.decode(real) as object,
|
||||
'not-the-server-secret',
|
||||
{ keyid: 'v2' },
|
||||
);
|
||||
const set = vi.spyOn(server.clients.redis, 'set');
|
||||
try {
|
||||
const session = uuidv4();
|
||||
await setToken(session, forged);
|
||||
await setToken(session, 'x'.repeat(100_000));
|
||||
expect(set).not.toHaveBeenCalled();
|
||||
|
||||
await setToken(session, real);
|
||||
expect(set).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
set.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves the token out of the warning when storing it fails', async () => {
|
||||
const { actor } = await makeUserAndActor();
|
||||
const token = await mintAppToken(actor, OPENER);
|
||||
const session = uuidv4();
|
||||
const failure = Object.assign(new Error('ERR unavailable'), {
|
||||
command: { name: 'set', args: [session, token, 'EX', '120'] },
|
||||
});
|
||||
const failingSet = vi
|
||||
.spyOn(server.clients.redis, 'set')
|
||||
.mockRejectedValueOnce(failure);
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const res = makeRes();
|
||||
await controller.loginSet(
|
||||
makeReq({ session, auth_token: token }),
|
||||
res,
|
||||
);
|
||||
expect(res.body).toEqual({ success: true });
|
||||
expect(warn).toHaveBeenCalled();
|
||||
const logged = warn.mock.calls
|
||||
.flat()
|
||||
.map((arg) => inspect(arg, { depth: 10 }))
|
||||
.join(' ');
|
||||
expect(logged).not.toContain(token);
|
||||
} finally {
|
||||
failingSet.mockRestore();
|
||||
warn.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('answers 400 when auth_token is not a string', async () => {
|
||||
await expect(
|
||||
controller.loginSet(
|
||||
makeReq({ session: uuidv4(), auth_token: { nested: true } }),
|
||||
makeRes(),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400, legacyCode: 'bad_request' });
|
||||
});
|
||||
});
|
||||
|
||||
// -- Batch grant / revoke + cross-app data grants -----------------------
|
||||
@@ -11023,3 +11628,100 @@ describe('AuthController — a seat address nobody confirmed', () => {
|
||||
expect(await emailOf(seat.id)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('AuthController account mail streams', () => {
|
||||
const spySend = () =>
|
||||
vi.spyOn(server.clients.email, 'send').mockResolvedValue(null);
|
||||
|
||||
it('sends password recovery on the auth stream', async () => {
|
||||
const { user } = await makeUserAndActor();
|
||||
const send = spySend();
|
||||
try {
|
||||
await controller.handleSendPassRecoveryEmail(
|
||||
makeReq({ email: user.email! }),
|
||||
makeRes(),
|
||||
);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
user.email,
|
||||
'email_password_recovery',
|
||||
expect.anything(),
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} finally {
|
||||
send.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('sends the password change notice on the auth stream', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const req = makeReq({ new_pass: 'a-fresh-password-456' }, { actor });
|
||||
(req as unknown as { userProtected: unknown }).userProtected = { user };
|
||||
const send = spySend();
|
||||
try {
|
||||
await controller.handleChangePassword(req, makeRes());
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
user.email,
|
||||
'password_change_notification',
|
||||
expect.anything(),
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} finally {
|
||||
send.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('sends both change-email messages on the auth stream', async () => {
|
||||
const { user, actor } = await makeUserAndActor();
|
||||
const newEmail = `cs_${uniq()}@test.local`;
|
||||
const send = spySend();
|
||||
const sendRaw = vi
|
||||
.spyOn(server.clients.email, 'sendRaw')
|
||||
.mockResolvedValue(null);
|
||||
try {
|
||||
await controller.handleChangeEmail(
|
||||
makeReq({ new_email: newEmail }, { actor }),
|
||||
makeRes(),
|
||||
);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
newEmail,
|
||||
'email_verification_link',
|
||||
expect.anything(),
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
expect(sendRaw).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ to: user.email, stream: 'auth' }),
|
||||
);
|
||||
} finally {
|
||||
send.mockRestore();
|
||||
sendRaw.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('sends the 2FA enable and disable notices on the auth stream', async () => {
|
||||
const { user, actor } = await makeUserAndActor({ email_confirmed: 1 });
|
||||
const send = spySend();
|
||||
try {
|
||||
const { secret } = await setup2fa(actor);
|
||||
await enable2fa(actor, { code: liveTotp(user.username, secret) });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
user.email,
|
||||
'enabled_2fa',
|
||||
{ username: user.username },
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
|
||||
await controller.handleDisable2fa(
|
||||
makeReq({ code: liveTotp(user.username, secret) }, { actor }),
|
||||
makeRes(),
|
||||
);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
user.email,
|
||||
'disabled_2fa',
|
||||
{ username: user.username },
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} finally {
|
||||
send.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -71,6 +71,7 @@ import { sessionCookieFlags } from '../../util/cookieFlags.js';
|
||||
import { cleanEmail, isStorableEmail } from '../../util/email.js';
|
||||
import { isGodmodeApp } from '../../util/godmodeApps.js';
|
||||
import { parsePhone } from '../../util/phone.js';
|
||||
import { deleteIfEqual } from '../../util/redisLock.js';
|
||||
import {
|
||||
bonusCodeInvalidError,
|
||||
checkSignupBonus,
|
||||
@@ -143,6 +144,12 @@ const DISPATCH_ID_MAX_LENGTH = 128;
|
||||
// One name for the flag, so the write and the read cannot drift apart.
|
||||
const APP_AUTHENTICATED_FLAG = 'flag:app-is-authenticated';
|
||||
|
||||
// Long enough to span the gap between one `/login/wait` and the client's next.
|
||||
const LOGIN_RELAY_TTL_SECONDS = 120;
|
||||
// Keyed by app too, so a token for one app never replaces another app's.
|
||||
const loginRelayKey = (session: string, appUid: string) =>
|
||||
`login:relay:${session}:${appUid}`;
|
||||
|
||||
// -- Post-login route limits -----------------------------------------
|
||||
//
|
||||
// The credential legs above (login, signup, recovery, confirmation) each
|
||||
@@ -345,36 +352,43 @@ export class AuthController extends PuterController {
|
||||
const expectedAppUid =
|
||||
await this.services.auth.appUidFromOrigin(reqOrigin);
|
||||
|
||||
const { resolve, promise } = Promise.withResolvers<void>();
|
||||
|
||||
let token: string | null = null;
|
||||
const listener = (_key: string, value: { authtoken: string }) => {
|
||||
token = value.authtoken;
|
||||
resolve();
|
||||
};
|
||||
this.clients.event.on(`pubsub.login.${session}`, listener);
|
||||
|
||||
const timeout = new Promise<void>((resolve) =>
|
||||
setTimeout(resolve, 10000),
|
||||
);
|
||||
await Promise.race([promise, timeout]);
|
||||
this.clients.event.off(`pubsub.login.${session}`, listener);
|
||||
if (!token) {
|
||||
throw new HttpError(408, 'Request timeout.', {
|
||||
legacyCode: 'request_timeout',
|
||||
});
|
||||
}
|
||||
const relayKey = loginRelayKey(session, expectedAppUid);
|
||||
const { resolve, promise } = Promise.withResolvers<string>();
|
||||
|
||||
// Audience check. The postMessage hand-off this relay stands in for
|
||||
// is origin-bound for free — it posts with `targetOrigin`, so a page
|
||||
// can only ever receive a token minted for *itself*. Delivering
|
||||
// server-side dropped that binding; this restores it. Without it a
|
||||
// popup talked into minting for app X (see `trustsOpenerOriginParam`
|
||||
// in the GUI) hands X's token to whoever holds the session id.
|
||||
if (!this.#tokenIsForApp(token, expectedAppUid)) {
|
||||
// Deliberately the same 408 the no-token path returns: a caller
|
||||
// learns only that nothing arrived for them, not that a token
|
||||
// for a different app went past.
|
||||
// server-side dropped that binding; this restores it. Any other token
|
||||
// is passed over and the wait runs on, so a caller learns only that
|
||||
// nothing arrived for them.
|
||||
const listener = (_key: string, value: { authtoken: string }) => {
|
||||
if (this.#relayedTokenAppUid(value.authtoken) === expectedAppUid) {
|
||||
resolve(value.authtoken);
|
||||
}
|
||||
};
|
||||
this.clients.event.on(`pubsub.login.${session}`, listener);
|
||||
let token: string | null;
|
||||
try {
|
||||
// Read after subscribing, so a token set before this wait is still delivered.
|
||||
token = await this.#takeStoredLoginToken(relayKey, expectedAppUid);
|
||||
if (!token) {
|
||||
const timeout = new Promise<null>((resolve) =>
|
||||
setTimeout(resolve, 10000, null),
|
||||
);
|
||||
token = await Promise.race([promise, timeout]);
|
||||
// A wait whose client went away leaves the copy for the next one.
|
||||
if (token && !req.socket?.destroyed) {
|
||||
await deleteIfEqual(
|
||||
this.clients.redis,
|
||||
relayKey,
|
||||
token,
|
||||
).catch(() => false);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.clients.event.off(`pubsub.login.${session}`, listener);
|
||||
}
|
||||
if (!token) {
|
||||
throw new HttpError(408, 'Request timeout.', {
|
||||
legacyCode: 'request_timeout',
|
||||
});
|
||||
@@ -386,24 +400,46 @@ export class AuthController extends PuterController {
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a relayed token is an app-under-user token minted for
|
||||
* `expectedAppUid`. Verifies the signature — an unverified decode would let
|
||||
* a caller relay a token whose claims it wrote itself.
|
||||
* Take the copy `/login/set` kept for this app, once. Null when there is
|
||||
* none, another wait took it, or Redis fails; the wait then just listens.
|
||||
*/
|
||||
#tokenIsForApp(token: string, expectedAppUid: string): boolean {
|
||||
async #takeStoredLoginToken(
|
||||
relayKey: string,
|
||||
expectedAppUid: string,
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const kept = await this.clients.redis.get(relayKey);
|
||||
if (!kept || this.#relayedTokenAppUid(kept) !== expectedAppUid) {
|
||||
return null;
|
||||
}
|
||||
const took = await deleteIfEqual(
|
||||
this.clients.redis,
|
||||
relayKey,
|
||||
kept,
|
||||
);
|
||||
return took ? kept : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The app a relayed app-under-user token was minted for, or null. Verifies
|
||||
* the signature — an unverified decode would let a caller relay a token
|
||||
* whose claims it wrote itself.
|
||||
*/
|
||||
#relayedTokenAppUid(token: string): string | null {
|
||||
try {
|
||||
const payload = this.services.token.verify<{
|
||||
type?: string;
|
||||
app_uid?: string;
|
||||
}>('auth', token);
|
||||
return (
|
||||
payload.type === 'app-under-user' &&
|
||||
!!payload.app_uid &&
|
||||
payload.app_uid === expectedAppUid
|
||||
);
|
||||
return payload.type === 'app-under-user' && payload.app_uid
|
||||
? payload.app_uid
|
||||
: null;
|
||||
} catch {
|
||||
// Malformed, expired, or signed with a key we don't hold.
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@Post('/login/set', {
|
||||
@@ -418,12 +454,34 @@ export class AuthController extends PuterController {
|
||||
})
|
||||
async loginSet(req: Request, res: Response) {
|
||||
const { session, auth_token } = req.body ?? {};
|
||||
if (!session || !auth_token || !validateUuid(session)) {
|
||||
if (
|
||||
!session ||
|
||||
typeof auth_token !== 'string' ||
|
||||
!auth_token ||
|
||||
!validateUuid(session)
|
||||
) {
|
||||
throw new HttpError(400, 'session and auth_token are required.', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
|
||||
// Only a token a wait could deliver is kept: one this server signed for an app.
|
||||
const appUid = this.#relayedTokenAppUid(auth_token);
|
||||
if (appUid) {
|
||||
try {
|
||||
await this.clients.redis.set(
|
||||
loginRelayKey(session, appUid),
|
||||
auth_token,
|
||||
'EX',
|
||||
LOGIN_RELAY_TTL_SECONDS,
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
'[login-set] storing the relayed token failed:',
|
||||
e instanceof Error ? e.message : String(e),
|
||||
);
|
||||
}
|
||||
}
|
||||
this.clients.event.emit(
|
||||
`pubsub.login.${session}`,
|
||||
{
|
||||
@@ -882,14 +940,19 @@ export class AuthController extends PuterController {
|
||||
if (!bonusCode) throw bonusCodeInvalidError();
|
||||
}
|
||||
|
||||
const user = await this.services.signup.signup({
|
||||
req,
|
||||
body,
|
||||
isTemp: is_temp,
|
||||
fingerprint,
|
||||
bonusCode,
|
||||
const { user, confirmationEmailSent } =
|
||||
await this.services.signup.signup({
|
||||
req,
|
||||
body,
|
||||
isTemp: is_temp,
|
||||
fingerprint,
|
||||
bonusCode,
|
||||
});
|
||||
await this.#completeLogin(req, res, user, undefined, {
|
||||
...(confirmationEmailSent !== undefined
|
||||
? { confirmation_email_sent: confirmationEmailSent }
|
||||
: {}),
|
||||
});
|
||||
await this.#completeLogin(req, res, user);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1028,10 +1091,12 @@ export class AuthController extends PuterController {
|
||||
email_confirm_code: code,
|
||||
});
|
||||
|
||||
await this.services.signup.sendConfirmationEmail(user, 'resend', {
|
||||
code,
|
||||
});
|
||||
res.json({});
|
||||
const sent = await this.services.signup.sendConfirmationEmail(
|
||||
user,
|
||||
'resend',
|
||||
{ code },
|
||||
);
|
||||
res.json({ sent });
|
||||
}
|
||||
|
||||
@Post('/confirm-email', {
|
||||
@@ -2019,6 +2084,7 @@ export class AuthController extends PuterController {
|
||||
user.email,
|
||||
'email_password_recovery',
|
||||
{ link },
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('[send-pass-recovery-email] send failed:', e);
|
||||
@@ -2282,6 +2348,7 @@ export class AuthController extends PuterController {
|
||||
{
|
||||
username: user.username,
|
||||
},
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('[change-password] notification send failed:', e);
|
||||
@@ -2507,6 +2574,7 @@ export class AuthController extends PuterController {
|
||||
new_email,
|
||||
'email_verification_link',
|
||||
{ link },
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('[change-email] new-address email failed:', e);
|
||||
@@ -2514,15 +2582,8 @@ export class AuthController extends PuterController {
|
||||
// Notify the old address too
|
||||
if (user.email) {
|
||||
try {
|
||||
await (
|
||||
this.clients.email as unknown as {
|
||||
sendRaw: (opts: {
|
||||
to: string;
|
||||
subject: string;
|
||||
text: string;
|
||||
}) => Promise<unknown>;
|
||||
}
|
||||
).sendRaw({
|
||||
await this.clients.email.sendRaw({
|
||||
stream: 'auth',
|
||||
to: user.email,
|
||||
subject: 'Your Puter email change was requested',
|
||||
text: `A change to ${new_email} was requested on your account. If this wasn't you, please contact support.`,
|
||||
@@ -2811,18 +2872,11 @@ export class AuthController extends PuterController {
|
||||
username,
|
||||
});
|
||||
|
||||
// Send confirmation email
|
||||
if (this.clients.email) {
|
||||
try {
|
||||
await this.clients.email.send(
|
||||
email,
|
||||
'email_verification_code',
|
||||
{ code: email_confirm_code },
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('[save-account] confirmation email failed:', e);
|
||||
}
|
||||
}
|
||||
await this.services.signup.sendConfirmationEmail(
|
||||
{ ...user, username, email },
|
||||
'save-account',
|
||||
{ code: email_confirm_code },
|
||||
);
|
||||
|
||||
try {
|
||||
this.clients.event?.emit(
|
||||
@@ -3585,21 +3639,18 @@ export class AuthController extends PuterController {
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
// `getByUuid` returns null when the row is missing, already
|
||||
// soft-revoked, or past `expires_at` — surface as 404 so a stale
|
||||
// manage-sessions UI doesn't 500 when it clicks revoke on a row
|
||||
// that already went away.
|
||||
const session = await this.stores.session.getByUuid(uuid);
|
||||
if (!session) {
|
||||
// Missing, already revoked and other users' rows all answer 404; an
|
||||
// expired row is still revoked so its children go with it.
|
||||
const session = await this.stores.session.getByUuidAny(uuid);
|
||||
if (
|
||||
!session ||
|
||||
session.revoked_at != null ||
|
||||
session.user_id !== req.actor!.user.id
|
||||
) {
|
||||
throw new HttpError(404, 'Session not found', {
|
||||
legacyCode: 'not_found',
|
||||
});
|
||||
}
|
||||
if (session.user_id !== req.actor!.user.id) {
|
||||
throw new HttpError(403, 'Can only revoke your own sessions', {
|
||||
legacyCode: 'unauthorized',
|
||||
});
|
||||
}
|
||||
await this.services.auth.revokeSession(uuid);
|
||||
const sessions = await this.services.auth.listSessions(req.actor!);
|
||||
res.json({ sessions });
|
||||
@@ -4275,9 +4326,12 @@ export class AuthController extends PuterController {
|
||||
|
||||
if (this.clients.email && user.email) {
|
||||
try {
|
||||
await this.clients.email.send(user.email, 'enabled_2fa', {
|
||||
username: user.username,
|
||||
});
|
||||
await this.clients.email.send(
|
||||
user.email,
|
||||
'enabled_2fa',
|
||||
{ username: user.username },
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('[configure-2fa] email send failed:', e);
|
||||
}
|
||||
@@ -4351,9 +4405,12 @@ export class AuthController extends PuterController {
|
||||
|
||||
if (this.clients.email && user.email) {
|
||||
try {
|
||||
await this.clients.email.send(user.email, 'disabled_2fa', {
|
||||
username: user.username,
|
||||
});
|
||||
await this.clients.email.send(
|
||||
user.email,
|
||||
'disabled_2fa',
|
||||
{ username: user.username },
|
||||
{ stream: 'auth' },
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('[disable-2fa] email send failed:', e);
|
||||
}
|
||||
@@ -4769,6 +4826,7 @@ export class AuthController extends PuterController {
|
||||
res: Response,
|
||||
user: UserRow,
|
||||
oidcContinuation?: unknown,
|
||||
extra: { confirmation_email_sent?: boolean } = {},
|
||||
): Promise<void> {
|
||||
const redirectUrl =
|
||||
oidcContinuation === undefined
|
||||
@@ -4803,6 +4861,7 @@ export class AuthController extends PuterController {
|
||||
token: gui_token,
|
||||
user: details,
|
||||
...(redirectUrl ? { redirectUrl } : {}),
|
||||
...extra,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -25,17 +25,13 @@ const ALLOWED_SORT_BY = ['name', 'size', 'modified', 'type'];
|
||||
const ALLOWED_SORT_ORDER = ['asc', 'desc'];
|
||||
|
||||
/**
|
||||
* Desktop/UI preference routes.
|
||||
*
|
||||
* Two categories:
|
||||
*
|
||||
* - User-level: taskbar items (UserStore)
|
||||
* - Folder-level: layout, sort_by/sort_order (fsentries table)
|
||||
* Folder view preference routes: layout and sort_by/sort_order, stored on the
|
||||
* folder's fsentries row.
|
||||
*/
|
||||
/**
|
||||
* Desktop preference writes — taskbar, layout, sort order. Each persists on
|
||||
* every call, and the GUI fires them on direct user action, so a per-minute
|
||||
* ceiling well above human speed is enough to catch a stuck client.
|
||||
* Desktop preference writes — layout, sort order. Each persists on every call,
|
||||
* and the GUI fires them on direct user action, so a per-minute ceiling well
|
||||
* above human speed is enough to catch a stuck client.
|
||||
*/
|
||||
const PREFERENCE_WRITE_LIMIT = {
|
||||
scope: 'desktop-preference',
|
||||
@@ -46,33 +42,6 @@ const PREFERENCE_WRITE_LIMIT = {
|
||||
|
||||
export class DesktopController extends PuterController {
|
||||
registerRoutes(router) {
|
||||
// -- Taskbar items -------------------------------------------
|
||||
|
||||
router.post(
|
||||
'/update-taskbar-items',
|
||||
{
|
||||
subdomain: 'api',
|
||||
requireUserActor: true,
|
||||
allowFullAccessToken: true,
|
||||
rateLimit: PREFERENCE_WRITE_LIMIT,
|
||||
},
|
||||
async (req, res) => {
|
||||
const { items } = req.body ?? {};
|
||||
if (!Array.isArray(items)) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Missing or invalid `items` array',
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
|
||||
await this.stores.user.update(req.actor.user.id, {
|
||||
taskbar_items: JSON.stringify(items),
|
||||
});
|
||||
res.json({});
|
||||
},
|
||||
);
|
||||
|
||||
// -- Folder layout -------------------------------------------
|
||||
|
||||
router.post(
|
||||
|
||||
@@ -136,50 +136,11 @@ describe('DesktopController routes', () => {
|
||||
router.routes.some((r) => r.path === '/set-desktop-bg'),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── /update-taskbar-items ───────────────────────────────────────────
|
||||
|
||||
describe('DesktopController POST /update-taskbar-items', () => {
|
||||
it('persists items as a JSON-encoded string', async () => {
|
||||
const { actor, userId } = await makeUser();
|
||||
const items = [{ name: 'editor' }, { name: 'browser' }];
|
||||
const { res } = makeRes();
|
||||
await callRoute(
|
||||
'post',
|
||||
'/update-taskbar-items',
|
||||
makeReq({ body: { items }, actor }),
|
||||
res,
|
||||
);
|
||||
|
||||
const refreshed = await server.stores.user.getById(userId);
|
||||
expect(refreshed?.taskbar_items).toBe(JSON.stringify(items));
|
||||
});
|
||||
|
||||
it('throws 400 when items is missing', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
callRoute(
|
||||
'post',
|
||||
'/update-taskbar-items',
|
||||
makeReq({ body: {}, actor }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('throws 400 when items is not an array', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
callRoute(
|
||||
'post',
|
||||
'/update-taskbar-items',
|
||||
makeReq({ body: { items: 'oops' }, actor }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
it('has no taskbar items route', () => {
|
||||
expect(
|
||||
router.routes.some((r) => r.path === '/update-taskbar-items'),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -324,7 +324,7 @@ describe('DriverController — concurrent acquire/release', () => {
|
||||
const driver = {
|
||||
...makeSyntheticDriver(),
|
||||
ping: async () => {
|
||||
Context.get('keepDriverSlot')?.(
|
||||
Context.get('keepConcurrencySlot')?.(
|
||||
new Promise<void>((resolve) => {
|
||||
settleWork = resolve;
|
||||
}),
|
||||
|
||||
@@ -51,7 +51,7 @@ import { PermissionUtil } from '../../services/permission/permissionUtil.js';
|
||||
import type { WithLifecycle } from '../../types';
|
||||
import { withSpan } from '../../util/span.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import { CallSlot } from './callSlot.js';
|
||||
import { CallSlot } from '../../core/http/callSlot.js';
|
||||
|
||||
type DriverInstance = WithLifecycle & Record<string, unknown>;
|
||||
|
||||
@@ -446,7 +446,7 @@ export class DriverController extends PuterController {
|
||||
});
|
||||
}
|
||||
slot.attach(handle, res);
|
||||
Context.set('keepDriverSlot', (work) => slot.keep(work));
|
||||
Context.set('keepConcurrencySlot', (work) => slot.keep(work));
|
||||
}
|
||||
|
||||
// Stash the requested driver name in Context so multi-provider
|
||||
|
||||
@@ -154,6 +154,26 @@ describe('GET /fs/readdir over HTTP', () => {
|
||||
).toEqual(['a', 'a/b']);
|
||||
});
|
||||
|
||||
it('answers 404 for a directory whose parent chain is broken', async () => {
|
||||
const { username, token } = env.users.user;
|
||||
const path = `/${username}/Documents/http-orphan-${Date.now()}`;
|
||||
const created = await fetch(new URL('/fs/mkdir', env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ path, auth_token: token }),
|
||||
});
|
||||
expect(created.status).toBe(200);
|
||||
const { uid } = (await created.json()) as { uid: string };
|
||||
await env.server.clients.db.write(
|
||||
'UPDATE fsentries SET path = NULL, parent_uid = ? WHERE uuid = ?',
|
||||
['does-not-exist', uid],
|
||||
);
|
||||
await env.server.stores.fsEntry.invalidateEntryCacheByUuid(uid);
|
||||
|
||||
const response = await fetch(readdirUrl({ uid, auth_token: token }));
|
||||
expect(response.status).toBe(404);
|
||||
});
|
||||
|
||||
it('mkdir does not return internal ids, storage columns or tokens', async () => {
|
||||
const { username, token } = env.users.user;
|
||||
const response = await fetch(new URL('/fs/mkdir', env.apiOrigin), {
|
||||
|
||||
@@ -1850,7 +1850,9 @@ export class FSController extends PuterController {
|
||||
}
|
||||
|
||||
async #assertCanDedupeCreate(actor: Actor, targetPath: string) {
|
||||
const existing = await this.stores.fsEntry.getEntryByPath(targetPath);
|
||||
const existing = await this.stores.fsEntry.getEntryByPath(targetPath, {
|
||||
foldedNames: true,
|
||||
});
|
||||
if (!existing) return;
|
||||
const parent = pathPosix.dirname(targetPath);
|
||||
await this.#assertAccess(
|
||||
|
||||
@@ -168,6 +168,27 @@ describe('legacy file downloads over HTTP', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it('GET /token-read types a file by its name, and one with no known extension as a byte stream', async () => {
|
||||
const typed = await requestFor(
|
||||
'/token-read',
|
||||
await writeFile('notes.txt', Buffer.from('hello')),
|
||||
);
|
||||
expect(typed.headers.get('content-type')).toBe(
|
||||
'text/plain; charset=utf-8',
|
||||
);
|
||||
|
||||
for (const name of ['README', 'archive.unknownext']) {
|
||||
const response = await requestFor(
|
||||
'/token-read',
|
||||
await writeFile(name, Buffer.from('hello')),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toBe(
|
||||
'application/octet-stream',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('GET /read answers byte_count on an empty file with an empty 200', async () => {
|
||||
const file = await writeFile('empty-byte-count.bin', Buffer.alloc(0));
|
||||
for (const query of ['byte_count=512', 'offset=0']) {
|
||||
|
||||
@@ -5111,9 +5111,8 @@ describe('LegacyFSController.updateFsentryThumbnail', () => {
|
||||
// ── GET /get-launch-apps ────────────────────────────────────────────
|
||||
//
|
||||
// `recent` is a launch-metadata producer: it returns `index_url` for
|
||||
// apps the user has opened before. The taskbar happens to launch these
|
||||
// by name (so AppDriver's hosted-backing guard applies), but the field
|
||||
// is served regardless, so the guard is applied here too.
|
||||
// apps the user has opened before. The field is served whoever reads it,
|
||||
// so AppDriver's hosted-backing guard is applied here too.
|
||||
|
||||
describe('LegacyFSController GET /get-launch-apps', () => {
|
||||
const hostedUrl = (sub: string) => `https://${sub}.site.puter.localhost/`;
|
||||
|
||||
@@ -401,11 +401,9 @@ export class LegacyFSController extends PuterController {
|
||||
.filter(Boolean);
|
||||
|
||||
// Don't hand out an index_url whose puter-hosted backing is
|
||||
// gone or reclaimed. The taskbar launches recents by name (so
|
||||
// AppDriver's guard applies), but this list is a
|
||||
// launch-metadata producer like any other — a future consumer
|
||||
// reading index_url straight off it shouldn't inherit a stale
|
||||
// origin.
|
||||
// gone or reclaimed. This list is a launch-metadata producer
|
||||
// like any other — a consumer reading index_url straight off
|
||||
// it shouldn't inherit a stale origin.
|
||||
let backingGoneFlags: boolean[];
|
||||
try {
|
||||
backingGoneFlags =
|
||||
@@ -767,8 +765,10 @@ export class LegacyFSController extends PuterController {
|
||||
normalizedTarget,
|
||||
);
|
||||
if (dedupeName) {
|
||||
const existing =
|
||||
await this.stores.fsEntry.getEntryByPath(normalizedTarget);
|
||||
const existing = await this.stores.fsEntry.getEntryByPath(
|
||||
normalizedTarget,
|
||||
{ foldedNames: true },
|
||||
);
|
||||
if (existing) {
|
||||
const parent = pathPosix.dirname(normalizedTarget);
|
||||
await assertAccess(
|
||||
@@ -950,9 +950,7 @@ export class LegacyFSController extends PuterController {
|
||||
// Trash, and `null`/`{}` when restoring. See
|
||||
// `src/gui/src/helpers.js` → `window.move_items`.
|
||||
newMetadata: (body.new_metadata ?? undefined) as
|
||||
| Record<string, unknown>
|
||||
| null
|
||||
| undefined,
|
||||
Record<string, unknown> | null | undefined,
|
||||
});
|
||||
const overwrittenEntry = replacedByMove(overwriteTarget, moved, name);
|
||||
const oldPath = source.path;
|
||||
@@ -1324,7 +1322,7 @@ export class LegacyFSController extends PuterController {
|
||||
if (options.realMime) {
|
||||
res.setHeader(
|
||||
'Content-Type',
|
||||
contentTypeFromMime(entry.name) as string,
|
||||
contentTypeFromMime(entry.name) || 'application/octet-stream',
|
||||
);
|
||||
} else {
|
||||
res.setHeader('Content-Type', 'application/octet-stream');
|
||||
@@ -1426,8 +1424,7 @@ export class LegacyFSController extends PuterController {
|
||||
}
|
||||
|
||||
type SignedOrEmpty =
|
||||
| (SignedFile & { path?: string })
|
||||
| Record<string, never>;
|
||||
(SignedFile & { path?: string }) | Record<string, never>;
|
||||
const result: { signatures: SignedOrEmpty[]; token?: string } = {
|
||||
signatures: [],
|
||||
};
|
||||
@@ -2054,10 +2051,7 @@ export class LegacyFSController extends PuterController {
|
||||
const subjectRef = body.subject;
|
||||
const appRef = body.app;
|
||||
const mode = (getString(body, 'mode') ?? 'read') as
|
||||
| 'see'
|
||||
| 'list'
|
||||
| 'read'
|
||||
| 'write';
|
||||
'see' | 'list' | 'read' | 'write';
|
||||
if (!subjectRef || !appRef)
|
||||
throw new HttpError(400, '`subject` and `app` are required', {
|
||||
legacyCode: 'bad_request',
|
||||
|
||||
@@ -277,4 +277,124 @@ describe('writes inside a folder shared with write', () => {
|
||||
expect(JSON.stringify(res.json)).not.toContain(s.realDir);
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
describe('a file the recipient opened with an app', () => {
|
||||
const makeApp = (ownerUserId: number) =>
|
||||
(
|
||||
env.server.stores.app.create as unknown as (
|
||||
fields: Record<string, unknown>,
|
||||
opts: { ownerUserId: number },
|
||||
) => Promise<{ uid: string; id: number }>
|
||||
)(
|
||||
{
|
||||
name: `sfw-app-${Math.random().toString(36).slice(2, 9)}`,
|
||||
title: 'Shared folder app',
|
||||
index_url: `https://sfw-${Math.random().toString(36).slice(2, 9)}.test/`,
|
||||
},
|
||||
{ ownerUserId },
|
||||
);
|
||||
|
||||
/** Opens `note.txt` with a new app, as a launch does. */
|
||||
const openWithApp = async (
|
||||
s: Awaited<ReturnType<typeof shareAFolder>>,
|
||||
) => {
|
||||
const recipientId = (await env.server.stores.user.getByUsername(
|
||||
s.recipient.username,
|
||||
))!.id;
|
||||
const app = await makeApp(recipientId);
|
||||
const signed = await post('/sign', s.recipient.token, {
|
||||
items: [{ uid: s.noteUid, action: 'write' }],
|
||||
app_uid: app.uid,
|
||||
});
|
||||
expect(signed.status).toBe(200);
|
||||
return { app, appToken: signed.json!.token as string };
|
||||
};
|
||||
|
||||
const read = async (token: string, query: Record<string, string>) => {
|
||||
const res = await fetch(
|
||||
new URL(`/read?${new URLSearchParams(query)}`, env.apiOrigin),
|
||||
{ headers: { authorization: `Bearer ${token}` } },
|
||||
);
|
||||
return { status: res.status, text: await res.text() };
|
||||
};
|
||||
|
||||
const write = (token: string, path: string, content: string) =>
|
||||
post('/fs/write', token, {
|
||||
fileMetadata: {
|
||||
path,
|
||||
size: content.length,
|
||||
contentType: 'text/plain',
|
||||
overwrite: true,
|
||||
},
|
||||
fileContent: content,
|
||||
});
|
||||
|
||||
it('reads and saves it by path and uid', async () => {
|
||||
const s = await shareAFolder();
|
||||
const { appToken } = await openWithApp(s);
|
||||
const masked = `${s.maskedDir}/note.txt`;
|
||||
const real = `${s.realDir}/note.txt`;
|
||||
|
||||
expect((await write(appToken, masked, 'one')).status).toBe(200);
|
||||
expect(await read(appToken, { uid: s.noteUid })).toEqual({
|
||||
status: 200,
|
||||
text: 'one',
|
||||
});
|
||||
expect((await write(appToken, real, 'two')).status).toBe(200);
|
||||
expect(await read(appToken, { file: real })).toEqual({
|
||||
status: 200,
|
||||
text: 'two',
|
||||
});
|
||||
expect(
|
||||
(await post('/stat', appToken, { uid: s.noteUid })).status,
|
||||
).toBe(200);
|
||||
}, 180_000);
|
||||
|
||||
it('only reads it once the share drops to read', async () => {
|
||||
const s = await shareAFolder();
|
||||
const { appToken } = await openWithApp(s);
|
||||
const dir = await post('/stat', s.owner.token, { path: s.realDir });
|
||||
const reshared = await post('/share', s.owner.token, {
|
||||
recipients: [s.recipient.username],
|
||||
items: [{ uid: dir.json!.uid }],
|
||||
mode: 'read',
|
||||
});
|
||||
expect(reshared.status).toBe(200);
|
||||
|
||||
expect((await read(appToken, { uid: s.noteUid })).status).toBe(200);
|
||||
expect(
|
||||
(await write(appToken, `${s.maskedDir}/note.txt`, 'x')).status,
|
||||
).toBe(403);
|
||||
}, 180_000);
|
||||
|
||||
it('gives nothing to another app of the recipient', async () => {
|
||||
const s = await shareAFolder();
|
||||
const { appToken } = await openWithApp(s);
|
||||
const recipientId = (await env.server.stores.user.getByUsername(
|
||||
s.recipient.username,
|
||||
))!.id;
|
||||
const other = await makeApp(recipientId);
|
||||
const minted = await post(
|
||||
'/auth/get-user-app-token',
|
||||
s.recipient.token,
|
||||
{ app_uid: other.uid },
|
||||
);
|
||||
expect(minted.status).toBe(200);
|
||||
const otherToken = minted.json!.token as string;
|
||||
|
||||
// The same requests the opened app gets through.
|
||||
const real = `${s.realDir}/note.txt`;
|
||||
expect((await read(appToken, { uid: s.noteUid })).status).toBe(200);
|
||||
expect((await write(appToken, real, 'one')).status).toBe(200);
|
||||
|
||||
expect((await read(otherToken, { uid: s.noteUid })).status).toBe(
|
||||
404,
|
||||
);
|
||||
expect((await write(otherToken, real, 'x')).status).toBe(404);
|
||||
expect(await read(appToken, { uid: s.noteUid })).toEqual({
|
||||
status: 200,
|
||||
text: 'one',
|
||||
});
|
||||
}, 180_000);
|
||||
});
|
||||
});
|
||||
@@ -79,6 +79,7 @@ interface CapturedResponse {
|
||||
statusCode: number;
|
||||
body: unknown;
|
||||
contentType?: string;
|
||||
location?: string;
|
||||
}
|
||||
|
||||
const makeReq = (init: {
|
||||
@@ -87,6 +88,7 @@ const makeReq = (init: {
|
||||
actor?: Actor;
|
||||
hostname?: string;
|
||||
protocol?: string;
|
||||
originalUrl?: string;
|
||||
}): Request => {
|
||||
return {
|
||||
body: {},
|
||||
@@ -94,6 +96,7 @@ const makeReq = (init: {
|
||||
headers: {},
|
||||
params: init.params ?? {},
|
||||
path: init.path ?? '/',
|
||||
originalUrl: init.originalUrl ?? init.path ?? '/',
|
||||
hostname: init.hostname ?? 'test.local',
|
||||
protocol: init.protocol ?? 'http',
|
||||
actor: init.actor,
|
||||
@@ -131,6 +134,11 @@ const makeRes = () => {
|
||||
captured.contentType = value;
|
||||
return res;
|
||||
}),
|
||||
redirect: vi.fn((location: string) => {
|
||||
captured.statusCode = 302;
|
||||
captured.location = location;
|
||||
return res;
|
||||
}),
|
||||
};
|
||||
return { res: res as unknown as Response, captured };
|
||||
};
|
||||
@@ -214,23 +222,72 @@ describe('HomepageController shell routes', () => {
|
||||
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
|
||||
});
|
||||
|
||||
it('serves the shell on /settings, /dashboard, /desktop, /action, /@:username', async () => {
|
||||
for (const path of [
|
||||
'/settings',
|
||||
'/settings/*splat',
|
||||
'/dashboard',
|
||||
'/dashboard/',
|
||||
'/desktop',
|
||||
'/desktop/',
|
||||
'/action/*splat',
|
||||
'/@:username',
|
||||
]) {
|
||||
it('leaves guiTokenDigest out for a session cookie that is not a user session', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const appActor = {
|
||||
...actor,
|
||||
app: { id: 1, uid: 'app-test' },
|
||||
session: { uid: uuidv4() },
|
||||
} as unknown as Actor;
|
||||
const req = Object.assign(makeReq({ path: '/', actor: appActor }), {
|
||||
tokenSource: 'cookie',
|
||||
cookies: { puter_token_v2: 'legacy' },
|
||||
});
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute('get', '/', req, res);
|
||||
expect(String(captured.body)).not.toContain('guiTokenDigest');
|
||||
});
|
||||
|
||||
it('serves the shell on /dashboard and /action', async () => {
|
||||
for (const path of ['/dashboard', '/dashboard/', '/action/*splat']) {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute('get', path, makeReq({ path }), res);
|
||||
expect(typeof captured.body).toBe('string');
|
||||
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
|
||||
}
|
||||
});
|
||||
|
||||
it('redirects /desktop to the root, keeping the query', async () => {
|
||||
for (const path of ['/desktop', '/desktop/']) {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute(
|
||||
'get',
|
||||
path,
|
||||
makeReq({ path, originalUrl: `${path}?ref=abc&x=1` }),
|
||||
res,
|
||||
);
|
||||
expect(captured.statusCode).toBe(302);
|
||||
expect(captured.location).toBe('/?ref=abc&x=1');
|
||||
expect(captured.body).toBeUndefined();
|
||||
}
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute('get', '/desktop', makeReq({ path: '/desktop' }), res);
|
||||
expect(captured.location).toBe('/');
|
||||
});
|
||||
|
||||
it('redirects /settings to the dashboard Home tab', async () => {
|
||||
for (const path of ['/settings', '/settings/*splat']) {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute('get', path, makeReq({ path }), res);
|
||||
expect(captured.statusCode).toBe(302);
|
||||
expect(captured.location).toBe('/dashboard#home');
|
||||
expect(captured.body).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it('redirects /@:username to the dashboard', async () => {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute(
|
||||
'get',
|
||||
'/@:username',
|
||||
makeReq({ path: '/@alice', params: { username: 'alice' } }),
|
||||
res,
|
||||
);
|
||||
expect(captured.statusCode).toBe(302);
|
||||
expect(captured.location).toBe('/');
|
||||
expect(captured.body).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── /app/:name ──────────────────────────────────────────────────────
|
||||
@@ -360,47 +417,21 @@ describe('HomepageController GET /app/:name', () => {
|
||||
expect(html).not.toContain('owner_user_id');
|
||||
});
|
||||
|
||||
it('serves the same app shell under /desktop/app/:name', async () => {
|
||||
const { userId } = await makeUser();
|
||||
const name = `desk-${Math.random().toString(36).slice(2, 10)}`;
|
||||
await server.stores.app.create(
|
||||
{
|
||||
name,
|
||||
title: 'Desktop App',
|
||||
description: 'opens on the desktop instead of the dashboard',
|
||||
index_url: `https://example.com/${name}/`,
|
||||
approved_for_listing: 1,
|
||||
},
|
||||
{ ownerUserId: userId },
|
||||
);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute(
|
||||
'get',
|
||||
'/desktop/app/:name',
|
||||
makeReq({ params: { name }, path: `/desktop/app/${name}` }),
|
||||
res,
|
||||
);
|
||||
|
||||
expect(captured.statusCode).toBe(200);
|
||||
const html = String(captured.body);
|
||||
expect(html).toMatch(/<!DOCTYPE html>/i);
|
||||
expect(html).toContain('Desktop App');
|
||||
});
|
||||
|
||||
it('returns 404 under /desktop/app/:name when the app is unknown', async () => {
|
||||
it('redirects /desktop/app/:name to /app/:name, keeping the query', async () => {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute(
|
||||
'get',
|
||||
'/desktop/app/:name',
|
||||
makeReq({
|
||||
params: { name: 'no-such-app' },
|
||||
path: '/desktop/app/no-such-app',
|
||||
params: { name: 'editor' },
|
||||
path: '/desktop/app/editor',
|
||||
originalUrl: '/desktop/app/editor?file=%2Falice%2Fa.txt&x=1',
|
||||
}),
|
||||
res,
|
||||
);
|
||||
expect(captured.statusCode).toBe(404);
|
||||
expect(String(captured.body)).toMatch(/<!DOCTYPE html>/i);
|
||||
expect(captured.statusCode).toBe(302);
|
||||
expect(captured.location).toBe('/app/editor?file=%2Falice%2Fa.txt&x=1');
|
||||
expect(captured.body).toBeUndefined();
|
||||
});
|
||||
|
||||
it('omits index_url even for a public app — the shell is not the launch authority', async () => {
|
||||
@@ -434,40 +465,39 @@ describe('HomepageController GET /app/:name', () => {
|
||||
// -- app landing rate limit --
|
||||
|
||||
describe('HomepageController app landing rate limit', () => {
|
||||
it('rate-limits /app/:name and /desktop/app/:name per network', () => {
|
||||
for (const path of ['/app/:name', '/desktop/app/:name']) {
|
||||
const route = router.routes.find(
|
||||
(r) => r.method === 'get' && r.path === path,
|
||||
);
|
||||
if (!route) throw new Error(`No GET ${path} route`);
|
||||
expect(route.options.rateLimit).toEqual({
|
||||
scope: 'app-landing',
|
||||
limit: 600,
|
||||
window: 60_000,
|
||||
key: 'ip',
|
||||
});
|
||||
}
|
||||
it('rate-limits /app/:name per network', () => {
|
||||
const route = router.routes.find(
|
||||
(r) => r.method === 'get' && r.path === '/app/:name',
|
||||
);
|
||||
if (!route) throw new Error('No GET /app/:name route');
|
||||
expect(route.options.rateLimit).toEqual({
|
||||
scope: 'app-landing',
|
||||
limit: 600,
|
||||
window: 60_000,
|
||||
key: 'ip',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── /show/* ─────────────────────────────────────────────────────────
|
||||
|
||||
describe('HomepageController GET /show/*splat', () => {
|
||||
it('emits a launch_app explorer call with the post-/show path', async () => {
|
||||
it('redirects to the path in the dashboard Files tab, keeping the query', async () => {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute(
|
||||
'get',
|
||||
'/show/*splat',
|
||||
makeReq({ path: '/show/alice/Documents' }),
|
||||
makeReq({
|
||||
path: '/show/alice/My%20Documents',
|
||||
originalUrl: '/show/alice/My%20Documents?ref=abc',
|
||||
}),
|
||||
res,
|
||||
);
|
||||
// The launch payload is JSON-serialized into the rendered HTML.
|
||||
// We assert the rendered shell included the explorer launch hint
|
||||
// pointing at the expected (slashed) path.
|
||||
const html = String(captured.body);
|
||||
expect(html).toContain('launch_app');
|
||||
expect(html).toContain('explorer');
|
||||
expect(html).toContain('/alice/Documents');
|
||||
expect(captured.statusCode).toBe(302);
|
||||
expect(captured.location).toBe(
|
||||
'/dashboard?ref=abc#files/alice/My%20Documents',
|
||||
);
|
||||
expect(captured.body).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
import express from 'express';
|
||||
import path from 'node:path';
|
||||
import { PuterController } from '../types.js';
|
||||
import { isPlainUserActor } from '../../core/actor.js';
|
||||
import { toAppShellView } from '../../util/appShellView.js';
|
||||
import type { PuterRouter } from '../../core/http/PuterRouter';
|
||||
import {
|
||||
@@ -27,11 +28,12 @@ import {
|
||||
subdomainLabelOf,
|
||||
} from '../../core/http/middleware/hostRedirects.js';
|
||||
import type { IConfig } from '../../types';
|
||||
import type { UserRow } from '../../stores/user/UserStore';
|
||||
import type {
|
||||
PuterHomepageService,
|
||||
PageMeta,
|
||||
LaunchOptions,
|
||||
PopupFrame,
|
||||
ShellContext,
|
||||
} from '../../services/homepage/PuterHomepageService';
|
||||
|
||||
/**
|
||||
@@ -170,7 +172,6 @@ export class HomepageController extends PuterController {
|
||||
req: express.Request,
|
||||
res: express.Response,
|
||||
metaOverrides: Partial<PageMeta> = {},
|
||||
launch: LaunchOptions = {},
|
||||
popupFrame?: PopupFrame,
|
||||
) => {
|
||||
const meta = { ...defaultMeta(req), ...metaOverrides };
|
||||
@@ -180,7 +181,16 @@ export class HomepageController extends PuterController {
|
||||
actor?: Parameters<typeof homepage.send>[0]['actor'];
|
||||
}
|
||||
).actor ?? null;
|
||||
await homepage.send({ req, res, actor }, meta, launch, popupFrame);
|
||||
await homepage.send(
|
||||
{
|
||||
req,
|
||||
res,
|
||||
actor,
|
||||
guiTokenDigest: this.#guiTokenDigest(req, actor),
|
||||
},
|
||||
meta,
|
||||
popupFrame,
|
||||
);
|
||||
};
|
||||
|
||||
// -- Root + path-aliased shell routes ------------------------
|
||||
@@ -252,7 +262,6 @@ export class HomepageController extends PuterController {
|
||||
req,
|
||||
res,
|
||||
{},
|
||||
{},
|
||||
{
|
||||
hostOrigin: embedHostOrigin(
|
||||
this.config,
|
||||
@@ -263,23 +272,42 @@ export class HomepageController extends PuterController {
|
||||
});
|
||||
}
|
||||
|
||||
router.get('/settings', {}, (req, res) => sendShell(req, res));
|
||||
router.get('/settings/*splat', {}, (req, res) => sendShell(req, res));
|
||||
// Old settings links land on the dashboard's Home tab.
|
||||
router.get('/settings', {}, (_req, res) =>
|
||||
res.redirect('/dashboard#home'),
|
||||
);
|
||||
router.get('/settings/*splat', {}, (_req, res) =>
|
||||
res.redirect('/dashboard#home'),
|
||||
);
|
||||
|
||||
router.get('/dashboard', {}, (req, res) => sendShell(req, res));
|
||||
router.get('/dashboard/', {}, (req, res) => sendShell(req, res));
|
||||
|
||||
router.get('/desktop', {}, (req, res) => sendShell(req, res));
|
||||
router.get('/desktop/', {}, (req, res) => sendShell(req, res));
|
||||
const queryOf = (req: express.Request) => {
|
||||
const separator = req.originalUrl.indexOf('?');
|
||||
return separator === -1 ? '' : req.originalUrl.slice(separator);
|
||||
};
|
||||
|
||||
// Old desktop links land on the dashboard, query intact.
|
||||
router.get('/desktop', {}, (req, res) =>
|
||||
res.redirect(`/${queryOf(req)}`),
|
||||
);
|
||||
router.get('/desktop/', {}, (req, res) =>
|
||||
res.redirect(`/${queryOf(req)}`),
|
||||
);
|
||||
|
||||
router.get('/action/*splat', {}, (req, res) => sendShell(req, res));
|
||||
|
||||
router.get('/@:username', {}, (req, res) => sendShell(req, res));
|
||||
// Old links to a user's public folder land on the dashboard.
|
||||
router.get('/@:username', {}, (_req, res) => res.redirect('/'));
|
||||
|
||||
// Old desktop app links open the app in the dashboard, query intact.
|
||||
router.get('/desktop/app/:name', {}, (req, res) => {
|
||||
const name = encodeURIComponent(String(req.params.name ?? ''));
|
||||
res.redirect(`/app/${name}${queryOf(req)}`);
|
||||
});
|
||||
|
||||
// -- /app/:name - app metadata baked into the shell ----------
|
||||
// `/desktop/app/:name` is the same landing booted on the desktop
|
||||
// instead of the dashboard; the GUI strips the prefix and treats the
|
||||
// rest of the path identically, so both render the same shell.
|
||||
|
||||
const sendAppShell = async (
|
||||
req: express.Request,
|
||||
@@ -334,26 +362,11 @@ export class HomepageController extends PuterController {
|
||||
{ rateLimit: APP_LANDING_LIMIT },
|
||||
sendAppShell,
|
||||
);
|
||||
router.get(
|
||||
'/desktop/app/:name',
|
||||
{ rateLimit: APP_LANDING_LIMIT },
|
||||
sendAppShell,
|
||||
);
|
||||
|
||||
// -- /show/* - launch explorer with the requested file path --
|
||||
|
||||
// Old /show links open the path in the dashboard's Files tab, query intact.
|
||||
router.get('/show/*splat', {}, (req, res) => {
|
||||
const filePath = req.path.slice('/show'.length);
|
||||
const launch: LaunchOptions = {
|
||||
on_initialized: [
|
||||
{
|
||||
$: 'window-call',
|
||||
fn_name: 'launch_app',
|
||||
args: [{ name: 'explorer', path: filePath }],
|
||||
},
|
||||
],
|
||||
};
|
||||
return sendShell(req, res, {}, launch);
|
||||
res.redirect(`/dashboard${queryOf(req)}#files${filePath}`);
|
||||
});
|
||||
|
||||
// -- /sw.js - the service worker, scoped to the whole origin --
|
||||
@@ -382,6 +395,38 @@ export class HomepageController extends PuterController {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lets the GUI skip `/get-gui-token`, which reads the session cookie: the
|
||||
* digest of the token it would hand over, null when there is no session
|
||||
* cookie (either name the auth probe reads), and undefined (the GUI asks)
|
||||
* when a cookie didn't resolve to a session.
|
||||
*/
|
||||
#guiTokenDigest(
|
||||
req: express.Request,
|
||||
actor: ShellContext['actor'],
|
||||
): string | null | undefined {
|
||||
if (req.tokenSource === 'cookie') {
|
||||
if (!actor || !isPlainUserActor(actor) || !actor.session?.uid) {
|
||||
return undefined;
|
||||
}
|
||||
return this.services.auth.guiTokenDigest(
|
||||
actor.user as UserRow,
|
||||
actor.session.uid,
|
||||
);
|
||||
}
|
||||
const cookies = (req.cookies ?? {}) as Record<string, unknown>;
|
||||
const hasSessionCookie = [
|
||||
this.config.cookie_name,
|
||||
'puter_token_v2',
|
||||
].some(
|
||||
(name) =>
|
||||
!!name &&
|
||||
typeof cookies[name] === 'string' &&
|
||||
cookies[name] !== '',
|
||||
);
|
||||
return hasSessionCookie ? undefined : null;
|
||||
}
|
||||
}
|
||||
|
||||
const safeJsonParse = (s: string): Record<string, unknown> | null => {
|
||||
|
||||
@@ -372,26 +372,6 @@ describe('OIDCController GET /auth/oidc/:provider/start', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('bakes a whitelisted desktop app landing (/desktop/app/<name>) into the state redirect_uri', async () => {
|
||||
const { res, captured } = makeRes();
|
||||
await callRoute(
|
||||
'get',
|
||||
'/auth/oidc/:provider/start',
|
||||
makeReq({
|
||||
params: { provider: 'custom' },
|
||||
query: { return_to: '/desktop/app/my-App_2' },
|
||||
}),
|
||||
res,
|
||||
);
|
||||
const state = new URL(captured.redirectUrl ?? '').searchParams.get(
|
||||
'state',
|
||||
);
|
||||
const decoded = oidc().verifyState(state!);
|
||||
expect(String(decoded?.redirect_uri)).toBe(
|
||||
`${TEST_ORIGIN}/desktop/app/my-App_2`,
|
||||
);
|
||||
});
|
||||
|
||||
it('ignores a non-whitelisted return_to', async () => {
|
||||
const bad_values = [
|
||||
'/app/evil/extra',
|
||||
@@ -400,6 +380,8 @@ describe('OIDCController GET /auth/oidc/:provider/start', () => {
|
||||
'//evil.test',
|
||||
'/settings',
|
||||
`/app/${'a'.repeat(101)}`,
|
||||
'/desktop',
|
||||
'/desktop/app/my-App_2',
|
||||
'/desktop/app/evil/extra',
|
||||
'/desktop/app/',
|
||||
'/dashboard/app/name',
|
||||
@@ -476,11 +458,11 @@ describe('OIDCController GET /auth/oidc/:provider/start', () => {
|
||||
sharedPath('b.txt'),
|
||||
sharedPath('a.txt'),
|
||||
],
|
||||
'/desktop',
|
||||
'/dashboard',
|
||||
),
|
||||
);
|
||||
const url = new URL(uri);
|
||||
expect(url.origin + url.pathname).toBe(`${TEST_ORIGIN}/desktop`);
|
||||
expect(url.origin + url.pathname).toBe(`${TEST_ORIGIN}/dashboard`);
|
||||
expect(url.searchParams.getAll('shared')).toEqual([
|
||||
sharedPath('a.txt'),
|
||||
sharedPath('b.txt'),
|
||||
|
||||
@@ -102,16 +102,14 @@ function signupOptionsFromState(
|
||||
}
|
||||
|
||||
// GUI pages an OIDC flow may return to: the root (where a share email lands),
|
||||
// /desktop, /dashboard, and direct app landings (/app/<name> and its
|
||||
// desktop-booted twin /desktop/app/<name>, mirroring APP_NAME_REGEX in
|
||||
// AppDriver). Strict whitelist — never a client-supplied URL (no open
|
||||
// redirect).
|
||||
// /dashboard, and direct app landings (/app/<name>, mirroring
|
||||
// APP_NAME_REGEX in AppDriver). Strict whitelist — never a client-supplied URL
|
||||
// (no open redirect).
|
||||
function isWhitelistedReturnPath(path: string): boolean {
|
||||
return (
|
||||
path === '/' ||
|
||||
path === '/desktop' ||
|
||||
path === '/dashboard' ||
|
||||
/^(\/desktop)?\/app\/[a-zA-Z0-9_-]{1,100}$/.test(path)
|
||||
/^\/app\/[a-zA-Z0-9_-]{1,100}$/.test(path)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -438,9 +436,8 @@ export class OIDCController extends PuterController {
|
||||
|
||||
let appRedirectUri = flowRedirects[flow] ?? (origin || '/');
|
||||
|
||||
// Optional GUI return path so login started from /desktop,
|
||||
// /dashboard, an /app/<name> landing, or a share link lands
|
||||
// back there.
|
||||
// Optional GUI return path so login started from /dashboard,
|
||||
// an /app/<name> landing, or a share link lands back there.
|
||||
const rawReturnTo = Array.isArray(req.query.return_to)
|
||||
? req.query.return_to[0]
|
||||
: req.query.return_to;
|
||||
|
||||
@@ -314,8 +314,8 @@ describe('OIDC account second-factor login', () => {
|
||||
|
||||
it('preserves ordinary OIDC login for accounts without 2FA', async () => {
|
||||
await account(false);
|
||||
const response = await callback('login', { return_to: '/desktop' });
|
||||
expect(response.headers.get('location')).toBe(`${env.origin}/desktop`);
|
||||
const response = await callback('login', { return_to: '/dashboard' });
|
||||
expect(response.headers.get('location')).toBe(`${env.origin}/dashboard`);
|
||||
expect(
|
||||
response.headers.get('set-cookie')?.includes('puter_auth_token='),
|
||||
).toBe(true);
|
||||
|
||||
@@ -24,11 +24,7 @@ import { computeNetworkFingerprint } from '../../core/http/middleware/rateLimit.
|
||||
import type { PuterRouter } from '../../core/http/PuterRouter.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import { PEER_COSTS } from './costs.js';
|
||||
import {
|
||||
readClaimedGrantIdentifier,
|
||||
signGuestGrant,
|
||||
verifyGuestGrant,
|
||||
} from './guestGrant.js';
|
||||
import { signGuestGrant, verifyGuestGrant } from './guestGrant.js';
|
||||
import {
|
||||
DEFAULT_FREE_SUBSCRIPTION,
|
||||
DEFAULT_TEMP_SUBSCRIPTION,
|
||||
@@ -191,19 +187,30 @@ export class PeerController extends PuterController {
|
||||
// guests share one bucket and no host can be relayed for by
|
||||
// more guests per minute than this — the only ceiling on guest
|
||||
// spend we can apply before the bytes are already spent.
|
||||
// A grant that doesn't parse can't name a bucket, so those
|
||||
// requests fall back to the caller's own network.
|
||||
// The limiter counts a request when it admits it, so only a
|
||||
// grant that would redeem names a host; otherwise anyone
|
||||
// holding the host's identifier could drain the bucket and
|
||||
// lock out every real guest. Anything else - forged, expired,
|
||||
// malformed, or no secret configured - is charged to the
|
||||
// caller's own network.
|
||||
rateLimit: {
|
||||
scope: 'peer-guest-turn',
|
||||
limit: 60,
|
||||
window: 60_000,
|
||||
key: (req: Request) => {
|
||||
const claimed = readClaimedGrantIdentifier(
|
||||
(req.body as { grant?: unknown } | undefined)
|
||||
?.grant,
|
||||
);
|
||||
return claimed
|
||||
? `host:${claimed}`
|
||||
const secret =
|
||||
this.config.peers?.guest_turn?.grant_secret;
|
||||
const verified = secret
|
||||
? verifyGuestGrant({
|
||||
grant: (
|
||||
req.body as
|
||||
{ grant?: unknown } | undefined
|
||||
)?.grant,
|
||||
secret,
|
||||
})
|
||||
: null;
|
||||
return verified?.status === 'ok'
|
||||
? `host:${verified.customIdentifier}`
|
||||
: `net:${computeNetworkFingerprint(req)}`;
|
||||
},
|
||||
},
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
import { toMicroCents } from '../../services/metering/utils';
|
||||
|
||||
// Microcents per byte of TURN egress ($0.05/GB).
|
||||
// Microcents per byte of TURN egress ($0.10/GB).
|
||||
export const PEER_COSTS = {
|
||||
'turn:egress-bytes': toMicroCents(0.1 / 1000 ** 3),
|
||||
} as const;
|
||||
@@ -1,7 +1,6 @@
|
||||
import { createHmac } from 'node:crypto';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
readClaimedGrantIdentifier,
|
||||
signGuestGrant,
|
||||
verifyGuestGrant,
|
||||
} from './guestGrant.js';
|
||||
@@ -243,59 +242,3 @@ describe('signGuestGrant / verifyGuestGrant', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('readClaimedGrantIdentifier', () => {
|
||||
it('reads the identifier from a valid grant', () => {
|
||||
const { grant } = signGuestGrant({
|
||||
customIdentifier: USER_ID,
|
||||
ttlSeconds: 60,
|
||||
secret: SECRET,
|
||||
});
|
||||
|
||||
expect(readClaimedGrantIdentifier(grant)).toBe(USER_ID);
|
||||
});
|
||||
|
||||
it('reads the claimed identifier without checking the signature', () => {
|
||||
// Bucketing runs before verification, so this is expected: a forged
|
||||
// grant still names a bucket, and the handler still rejects it.
|
||||
const grant = forgeGrant(
|
||||
{ id: USER_ID, exp: 9_999_999_999, n: 'x' },
|
||||
{ secret: 'wrong-secret' },
|
||||
);
|
||||
|
||||
expect(readClaimedGrantIdentifier(grant)).toBe(USER_ID);
|
||||
});
|
||||
|
||||
it('reads an expired grant, which the handler then rejects', () => {
|
||||
const { grant } = signGuestGrant({
|
||||
customIdentifier: USER_ID,
|
||||
ttlSeconds: 60,
|
||||
secret: SECRET,
|
||||
now: 1_000_000_000_000,
|
||||
});
|
||||
|
||||
expect(readClaimedGrantIdentifier(grant)).toBe(USER_ID);
|
||||
expect(verifyGuestGrant({ grant, secret: SECRET })).toEqual({
|
||||
status: 'expired',
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a non-string', 42],
|
||||
['undefined', undefined],
|
||||
['a wrong-shaped string', 'not-a-grant'],
|
||||
['an unknown version', 'pg2.abc.def'],
|
||||
['an over-long string', `pg1.${'a'.repeat(600)}.sig`],
|
||||
[
|
||||
'a non-JSON payload',
|
||||
`pg1.${Buffer.from('x').toString('base64url')}.sig`,
|
||||
],
|
||||
])('returns null for %s', (_label, grant) => {
|
||||
expect(readClaimedGrantIdentifier(grant)).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for an identifier of the wrong shape', () => {
|
||||
const grant = forgeGrant({ id: 'nope', exp: 9_999_999_999 });
|
||||
expect(readClaimedGrantIdentifier(grant)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -171,37 +171,3 @@ export const verifyGuestGrant = ({
|
||||
expiresAt: payload.exp,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Read the claimed identifier out of a grant _without_ verifying it, for
|
||||
* rate-limit bucketing only.
|
||||
*
|
||||
* Bucketing has to happen before the handler runs, and a forged grant is
|
||||
* rejected there without reaching the upstream API — so an unverified read is
|
||||
* enough to put a host's guests in one bucket, and the worst a forger achieves
|
||||
* is choosing which bucket their own rejections land in. Never use this to
|
||||
* decide attribution.
|
||||
*
|
||||
* @returns The claimed identifier, or null if the grant doesn't parse.
|
||||
*/
|
||||
export const readClaimedGrantIdentifier = (grant: unknown): string | null => {
|
||||
if (typeof grant !== 'string' || grant.length > MAX_GRANT_LENGTH) {
|
||||
return null;
|
||||
}
|
||||
const parts = grant.split('.');
|
||||
if (parts.length !== 3 || parts[0] !== GRANT_VERSION) return null;
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(parts[1]!, 'base64url').toString('utf8'),
|
||||
) as { id?: unknown };
|
||||
if (
|
||||
typeof payload?.id !== 'string' ||
|
||||
!IDENTIFIER_RE.test(payload.id)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return payload.id;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
@@ -44,7 +44,7 @@ import {
|
||||
} from 'vitest';
|
||||
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import type { ChatCompletionDriver } from '../../drivers/ai-chat/ChatCompletionDriver.js';
|
||||
import type { ChatCompletionDriver } from '../../drivers/ai/interfaces/chat/ChatCompletionDriver.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { inRequestScope, setupTestServer } from '../../testUtil.js';
|
||||
import { PuterAIController } from './PuterAIController.js';
|
||||
|
||||
@@ -49,8 +49,8 @@ import type { RouteOptions } from '../../core/http/index.js';
|
||||
import {
|
||||
COUNT_TOKENS,
|
||||
type ChatCompletionDriver,
|
||||
} from '../../drivers/ai-chat/ChatCompletionDriver.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../../drivers/util/aiLimits.js';
|
||||
} from '../../drivers/ai/interfaces/chat/ChatCompletionDriver.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../../drivers/ai/shared/aiLimits.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { inRequestScope, setupTestServer } from '../../testUtil.js';
|
||||
import { PuterAIController } from './PuterAIController.js';
|
||||
|
||||
@@ -30,25 +30,28 @@ import type { PuterRouter } from '../../core/http/PuterRouter.js';
|
||||
import {
|
||||
COUNT_TOKENS,
|
||||
type ChatCompletionDriver,
|
||||
} from '../../drivers/ai-chat/ChatCompletionDriver.js';
|
||||
} from '../../drivers/ai/interfaces/chat/ChatCompletionDriver.js';
|
||||
import type {
|
||||
IChatCompleteResult,
|
||||
IChatMessageResult,
|
||||
IChatModel,
|
||||
ICompleteArguments,
|
||||
UsageDetails,
|
||||
} from '../../drivers/ai-chat/types.js';
|
||||
} from '../../drivers/ai/interfaces/chat/types.js';
|
||||
import {
|
||||
outputFormatFromResponseFormat,
|
||||
rejectStatefulResponsesFields,
|
||||
toolChoiceFromWire,
|
||||
} from '../../drivers/ai-chat/utils/openaiParams.js';
|
||||
} from '../../drivers/ai/protocols/openaiCompatible/openaiParams.js';
|
||||
import {
|
||||
promoteStopForToolCalls,
|
||||
toFinishReason,
|
||||
} from '../../drivers/ai-chat/utils/stopReason.js';
|
||||
} from '../../drivers/ai/interfaces/chat/utils/stopReason.js';
|
||||
import { isDriverStreamResult } from '../../drivers/meta.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../../drivers/util/aiLimits.js';
|
||||
import {
|
||||
AI_CONCURRENT,
|
||||
AI_RATE_LIMIT,
|
||||
} from '../../drivers/ai/shared/aiLimits.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import { parseAnthropicRequest, toAnthropicMessage } from './anthropicWire.js';
|
||||
import { pipeNdjsonStream } from './ndjsonStream.js';
|
||||
@@ -1301,13 +1304,18 @@ export class PuterAIController extends PuterController {
|
||||
return driver;
|
||||
}
|
||||
|
||||
/** A completion that stops, and stops billing, when the caller hangs up. */
|
||||
/**
|
||||
* A completion whose caller may hang up. It runs on to its usage report
|
||||
* either way, so it holds the route's concurrency slot until it settles.
|
||||
*/
|
||||
#complete(
|
||||
res: Response,
|
||||
args: ICompleteArguments,
|
||||
): Promise<IChatCompleteResult> {
|
||||
Context.set('abortSignal', abortOnDisconnect(res));
|
||||
return this.#driver().complete(args);
|
||||
const completion = this.#driver().complete(args);
|
||||
Context.get('keepConcurrencySlot')?.(completion);
|
||||
return completion;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { IChatMessageResult } from '../../drivers/ai-chat/types.js';
|
||||
import type { IChatMessageResult } from '../../drivers/ai/interfaces/chat/types.js';
|
||||
import {
|
||||
anthropicUsage,
|
||||
parseAnthropicRequest,
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
|
||||
import type { IncomingHttpHeaders } from 'node:http';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { sanitizeCacheControl } from '../../drivers/ai-chat/providers/claude/anthropicPolicy.js';
|
||||
import { sanitizeCacheControl } from '../../drivers/ai/providers/claude/chat/anthropicPolicy.js';
|
||||
import type {
|
||||
ICompleteArguments,
|
||||
IChatMessageResult,
|
||||
@@ -42,9 +42,9 @@ import type {
|
||||
ThinkingConfig,
|
||||
ToolChoice,
|
||||
UsageDetails,
|
||||
} from '../../drivers/ai-chat/types.js';
|
||||
import { needsOpenAICoercion } from '../../drivers/ai-chat/utils/normalizeToOpenAI.js';
|
||||
import { fromFinishReason } from '../../drivers/ai-chat/utils/stopReason.js';
|
||||
} from '../../drivers/ai/interfaces/chat/types.js';
|
||||
import { needsOpenAICoercion } from '../../drivers/ai/interfaces/chat/utils/normalizeToOpenAI.js';
|
||||
import { fromFinishReason } from '../../drivers/ai/interfaces/chat/utils/stopReason.js';
|
||||
|
||||
const toStringOrEmpty = (v: unknown): string =>
|
||||
typeof v === 'string' ? v : '';
|
||||
|
||||
@@ -20,14 +20,14 @@
|
||||
/**
|
||||
* SSE transport (`startSse`) plus the Anthropic `/v1/messages` streaming writer
|
||||
* (`AnthropicSseWriter`), which turns the driver's NDJSON chunk protocol
|
||||
* (`drivers/ai-chat/utils/Streaming.js`) into the exact Anthropic event
|
||||
* sequence: `message_start` → `content_block_start/delta/stop`* →
|
||||
* (`drivers/ai/interfaces/chat/utils/Streaming.js`) into the exact Anthropic
|
||||
* event sequence: `message_start` → `content_block_start/delta/stop`* →
|
||||
* `message_delta` → `message_stop`.
|
||||
*/
|
||||
|
||||
import type { Response } from 'express';
|
||||
import type { UsageDetails } from '../../drivers/ai-chat/types.js';
|
||||
import { promoteStopForToolCalls } from '../../drivers/ai-chat/utils/stopReason.js';
|
||||
import type { UsageDetails } from '../../drivers/ai/interfaces/chat/types.js';
|
||||
import { promoteStopForToolCalls } from '../../drivers/ai/interfaces/chat/utils/stopReason.js';
|
||||
import { anthropicUsage } from './anthropicWire.js';
|
||||
|
||||
// -- SSE transport ------------------------------------------------------------
|
||||
|
||||
@@ -301,6 +301,7 @@ export class SystemController extends PuterController {
|
||||
if (this.clients.email && req.actor.user?.email) {
|
||||
try {
|
||||
await this.clients.email.sendRaw({
|
||||
stream: 'notify',
|
||||
to: supportEmail,
|
||||
replyTo: req.actor.user.email,
|
||||
subject: `Contact from ${req.actor.user.username}`,
|
||||
|
||||
@@ -547,6 +547,7 @@ describe('SystemController POST /contactUs — attachments', () => {
|
||||
|
||||
expect(sendRaw).toHaveBeenCalledTimes(1);
|
||||
const sent = sendRaw.mock.calls[0][0] as {
|
||||
stream: string;
|
||||
text: string;
|
||||
attachments: Array<{
|
||||
filename: string;
|
||||
@@ -555,6 +556,7 @@ describe('SystemController POST /contactUs — attachments', () => {
|
||||
contentDisposition: string;
|
||||
}>;
|
||||
};
|
||||
expect(sent.stream).toBe('notify');
|
||||
expect(sent.attachments).toHaveLength(1);
|
||||
expect(sent.attachments[0].filename).toBe('payload.png');
|
||||
expect(sent.attachments[0].contentType).toBe('image/png');
|
||||
|
||||
@@ -813,7 +813,10 @@ export class WebDAVController extends PuterController {
|
||||
const userId = actor.user!.id as number;
|
||||
await this.#assertCreate(actor, davPath);
|
||||
|
||||
const existing = await this.stores.fsEntry.getEntryByPath(davPath);
|
||||
// A name only the collation matches is taken too.
|
||||
const existing = await this.stores.fsEntry.getEntryByPath(davPath, {
|
||||
foldedNames: true,
|
||||
});
|
||||
if (existing)
|
||||
throw new HttpError(405, 'Already exists', {
|
||||
legacyCode: 'bad_request',
|
||||
@@ -937,7 +940,9 @@ export class WebDAVController extends PuterController {
|
||||
await this.#assertCreate(actor, destPath);
|
||||
|
||||
const overwrite = req.headers.overwrite !== 'F';
|
||||
const destExists = await this.stores.fsEntry.getEntryByPath(destPath);
|
||||
const destExists = await this.stores.fsEntry.getEntryByPath(destPath, {
|
||||
foldedNames: true,
|
||||
});
|
||||
if (destExists && !overwrite)
|
||||
throw new HttpError(412, 'Destination exists and Overwrite=F', {
|
||||
legacyCode: 'conflict',
|
||||
@@ -985,7 +990,9 @@ export class WebDAVController extends PuterController {
|
||||
await this.#assertCreate(actor, destPath);
|
||||
|
||||
const overwrite = req.headers.overwrite !== 'F';
|
||||
const destExists = await this.stores.fsEntry.getEntryByPath(destPath);
|
||||
const destExists = await this.stores.fsEntry.getEntryByPath(destPath, {
|
||||
foldedNames: true,
|
||||
});
|
||||
if (destExists && !overwrite)
|
||||
throw new HttpError(412, 'Destination exists and Overwrite=F', {
|
||||
legacyCode: 'conflict',
|
||||
|
||||
@@ -243,6 +243,33 @@ describe('WispController', () => {
|
||||
expect(await verifyStatus(token)).toBe(403);
|
||||
});
|
||||
|
||||
it('binds a token minted by an access token to that token while the replica lags', async () => {
|
||||
const { sessionActor, apiTokenActor, apiToken } = await freshUser();
|
||||
const db = server.clients.db;
|
||||
const original = db.read.bind(db);
|
||||
const spy = vi
|
||||
.spyOn(db, 'read')
|
||||
.mockImplementation(async (sql: string, params?: unknown[]) =>
|
||||
sql.includes("FROM `sessions` WHERE `kind` = 'access_token'")
|
||||
? []
|
||||
: original(sql, params),
|
||||
);
|
||||
let token: string;
|
||||
try {
|
||||
token = await relayTokenFor(apiTokenActor);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
expect(await verifyStatus(token)).toBe(200);
|
||||
|
||||
await server.services.auth.revokeAccessToken(
|
||||
sessionActor,
|
||||
apiToken,
|
||||
);
|
||||
|
||||
expect(await verifyStatus(token)).toBe(403);
|
||||
});
|
||||
|
||||
it('applies the account checks to a token with no session claim', async () => {
|
||||
const { row } = await freshUser();
|
||||
const token = server.services.token.sign(
|
||||
|
||||
@@ -151,11 +151,12 @@ export class WispController extends PuterController {
|
||||
|
||||
/**
|
||||
* The session a relay token lives no longer than. An access token's is its
|
||||
* own session row; the actor doesn't carry it.
|
||||
* own session row.
|
||||
*/
|
||||
async #sessionUidFor(actor: Actor): Promise<string | undefined> {
|
||||
if (actor.session?.uid) return actor.session.uid;
|
||||
if (!actor.accessToken) return undefined;
|
||||
if (actor.accessToken.sessionUid) return actor.accessToken.sessionUid;
|
||||
const row = await this.stores.session.findActiveByAccessTokenUid(
|
||||
actor.accessToken.uid,
|
||||
);
|
||||
|
||||
@@ -47,6 +47,11 @@ export interface ActorAccessToken {
|
||||
* reach while revokes, audits and gate hooks can still name the app.
|
||||
*/
|
||||
godmodeApp?: ActorApp | null;
|
||||
/**
|
||||
* The token's own session row, checked live when it authenticated. Unset
|
||||
* when the token carries no `session_uid` naming its row.
|
||||
*/
|
||||
sessionUid?: string;
|
||||
}
|
||||
|
||||
export interface Actor {
|
||||
|
||||
@@ -69,11 +69,11 @@ export interface KnownContextFields {
|
||||
*/
|
||||
abortSignal: AbortSignal;
|
||||
/**
|
||||
* Keeps the call's concurrency slot until `work` settles (set by
|
||||
* DriverController when the method is capped), for work that outlives the
|
||||
* response.
|
||||
* Keeps the request's concurrency slot until `work` settles, for work that
|
||||
* outlives the response. Set by DriverController for a capped method and by
|
||||
* the route concurrency gate.
|
||||
*/
|
||||
keepDriverSlot: (work: Promise<unknown>) => void;
|
||||
keepConcurrencySlot: (work: Promise<unknown>) => void;
|
||||
/**
|
||||
* Aborts the chat driver's upstream request. A client disconnect doesn't
|
||||
* fire it: the upstream bills a request it has accepted either way, so the
|
||||
@@ -102,7 +102,7 @@ const KNOWN_KEY_FLAGS: Record<keyof KnownContextFields, true> = {
|
||||
requestId: true,
|
||||
driverName: true,
|
||||
abortSignal: true,
|
||||
keepDriverSlot: true,
|
||||
keepConcurrencySlot: true,
|
||||
upstreamSignal: true,
|
||||
strictUpstreamErrors: true,
|
||||
platformWrite: true,
|
||||
|
||||
File renamed without changes.
@@ -28,9 +28,9 @@ interface SlotHandle {
|
||||
}
|
||||
|
||||
/**
|
||||
* A driver call's concurrency slot, held until the response is done and the
|
||||
* call's work has settled, including work a driver carries on after the caller
|
||||
* hung up or was answered.
|
||||
* A request's concurrency slot, held until the response is done and the work
|
||||
* registered with `keep` has settled, including work that carries on after the
|
||||
* caller hung up or was answered.
|
||||
*/
|
||||
export class CallSlot {
|
||||
#handle?: SlotHandle;
|
||||
@@ -50,6 +50,13 @@ declare global {
|
||||
|
||||
tokenAuthFailed?: boolean;
|
||||
|
||||
/**
|
||||
* What `authenticate` threw while checking the token (a database
|
||||
* failure, say). The token may be fine; gates rethrow this instead
|
||||
* of answering 401.
|
||||
*/
|
||||
tokenAuthError?: unknown;
|
||||
|
||||
/**
|
||||
* Set when a token authenticated but its app is on the origin
|
||||
* blocklist. The auth probe leaves `actor` unset; gates translate
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../../testUtil.js';
|
||||
|
||||
const loadShed = () =>
|
||||
Object.assign(new Error('Database operation failed'), {
|
||||
code: 'dbBatchFailed',
|
||||
});
|
||||
|
||||
/**
|
||||
* A token whose lookup fails is not a dead token: clients discard a token on
|
||||
* 401 `token_auth_failed`, so that answer is kept for tokens that checked out
|
||||
* invalid.
|
||||
*/
|
||||
describe('auth probe when the token lookup fails', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const get = (path: string, token: string) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
/** Run `fn` while the next user lookup rejects with `error`. */
|
||||
const withFailedUserLookup = async <T>(
|
||||
error: Error,
|
||||
fn: () => Promise<T>,
|
||||
): Promise<T> => {
|
||||
const spy = vi
|
||||
.spyOn(env.server.stores.user, 'getByUuid')
|
||||
.mockRejectedValueOnce(error);
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
};
|
||||
|
||||
it.each([
|
||||
['a session token', () => env.users.user.token],
|
||||
['an access token', () => env.users.user.apiToken],
|
||||
])(
|
||||
'answers %s with 503 db_unavailable during a database outage',
|
||||
async (_kind, tokenOf) => {
|
||||
const token = tokenOf();
|
||||
const res = await withFailedUserLookup(loadShed(), () =>
|
||||
get('/whoami', token),
|
||||
);
|
||||
expect(res.status).toBe(503);
|
||||
expect(res.headers.get('retry-after')).toBe('5');
|
||||
expect(await res.json()).toEqual({
|
||||
error: 'Service temporarily unavailable',
|
||||
message: 'Service temporarily unavailable',
|
||||
code: 'db_unavailable',
|
||||
});
|
||||
|
||||
expect((await get('/whoami', token)).status).toBe(200);
|
||||
},
|
||||
);
|
||||
|
||||
it('answers an unexpected lookup failure with a generic 500, not a 401', async () => {
|
||||
const res = await withFailedUserLookup(new Error('boom'), () =>
|
||||
get('/whoami', env.users.user.token),
|
||||
);
|
||||
expect(res.status).toBe(500);
|
||||
expect((await res.json()).code).toBe('internal_error');
|
||||
});
|
||||
|
||||
it('serves a route that does not require auth anonymously', async () => {
|
||||
const res = await withFailedUserLookup(loadShed(), () =>
|
||||
get('/version', env.users.user.token),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it('still answers an invalid token with 401 token_auth_failed', async () => {
|
||||
const res = await get('/whoami', 'not-a-real-jwt');
|
||||
expect(res.status).toBe(401);
|
||||
expect(await res.json()).toEqual({
|
||||
error: 'Authentication failed',
|
||||
message: 'Authentication failed',
|
||||
code: 'token_auth_failed',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -555,17 +555,18 @@ describe('createAuthProbe — actor attachment + failure tracking', () => {
|
||||
expect(req.tokenAuthFailed).toBe(true);
|
||||
});
|
||||
|
||||
it('never rejects — sets tokenAuthFailed even when AuthService throws', async () => {
|
||||
it('never rejects — records the error, not tokenAuthFailed, when AuthService throws', async () => {
|
||||
const stub = makeStubAuth();
|
||||
stub.setNext('throw');
|
||||
const probe = createAuthProbe({ authService: stub.service });
|
||||
const { req, next } = await runProbe(
|
||||
probe,
|
||||
makeReq({ headers: { authorization: 'Bearer bad-tok' } }),
|
||||
makeReq({ headers: { authorization: 'Bearer some-tok' } }),
|
||||
);
|
||||
// Critical invariant: the probe NEVER rejects, no matter what.
|
||||
expect(next).toHaveBeenCalledWith();
|
||||
expect(req.tokenAuthFailed).toBe(true);
|
||||
expect(req.tokenAuthFailed).toBeUndefined();
|
||||
expect(req.tokenAuthError).toBeInstanceOf(Error);
|
||||
expect(req.actor).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -880,4 +881,27 @@ describe('createAuthProbe (integration) — real session token → real actor',
|
||||
expect(req.actor).toBeUndefined();
|
||||
expect(req.tokenAuthFailed).toBe(true);
|
||||
});
|
||||
|
||||
it('records a failed user lookup on a valid token without marking it invalid', async () => {
|
||||
const user = await makeUser();
|
||||
const { token } = await authService.createSessionToken(user);
|
||||
const lookupError = Object.assign(new Error('db down'), {
|
||||
code: 'dbBatchFailed',
|
||||
});
|
||||
const spy = vi
|
||||
.spyOn(server.stores.user, 'getByUuid')
|
||||
.mockRejectedValueOnce(lookupError);
|
||||
try {
|
||||
const probe = createAuthProbe({ authService });
|
||||
const { req } = await runProbe(
|
||||
probe,
|
||||
makeReq({ headers: { authorization: `Bearer ${token}` } }),
|
||||
);
|
||||
expect(req.actor).toBeUndefined();
|
||||
expect(req.tokenAuthFailed).toBeUndefined();
|
||||
expect(req.tokenAuthError).toBe(lookupError);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -45,9 +45,9 @@ interface AuthProbeOptions {
|
||||
* and valid — attaches an `Actor` to `req.actor`.
|
||||
*
|
||||
* Key property: this middleware **never rejects**. Missing tokens, malformed
|
||||
* tokens, expired tokens, tokens pointing at deleted users — all result in
|
||||
* `req.actor` being left undefined. Per-route gates decide whether absence is
|
||||
* acceptable.
|
||||
* tokens, expired tokens, tokens pointing at deleted users, and lookups that
|
||||
* fail outright all result in `req.actor` being left undefined. Per-route gates
|
||||
* decide whether absence is acceptable.
|
||||
*
|
||||
* Token lookup order:
|
||||
*
|
||||
@@ -153,8 +153,10 @@ export const createAuthProbe = (opts: AuthProbeOptions): RequestHandler => {
|
||||
} else if (result.invalid) {
|
||||
req.tokenAuthFailed = true;
|
||||
}
|
||||
} catch {
|
||||
req.tokenAuthFailed = true;
|
||||
} catch (err) {
|
||||
// Not a verdict on the token: routes that require auth answer
|
||||
// with this error, routes that don't serve the request anonymously.
|
||||
req.tokenAuthError = err;
|
||||
}
|
||||
next();
|
||||
};
|
||||
|
||||
@@ -288,6 +288,22 @@ describe('requireAuthGate', () => {
|
||||
});
|
||||
expectHttpError(got, 401, 'reauth_required');
|
||||
});
|
||||
|
||||
it('passes on the error when the token could not be checked', () => {
|
||||
const lookupError = Object.assign(new Error('down'), {
|
||||
code: 'dbBatchFailed',
|
||||
});
|
||||
const got = runGate(requireAuthGate(), { tokenAuthError: lookupError });
|
||||
expect(got).toBe(lookupError);
|
||||
});
|
||||
|
||||
it('still answers token_auth_failed when a check found the token invalid', () => {
|
||||
const got = runGate(requireAuthGate(), {
|
||||
tokenAuthFailed: true,
|
||||
tokenAuthError: new Error('down'),
|
||||
});
|
||||
expectHttpError(got, 401, 'token_auth_failed');
|
||||
});
|
||||
});
|
||||
|
||||
// ── requireUserActorGate ────────────────────────────────────────────
|
||||
@@ -330,6 +346,14 @@ describe('requireUserActorGate', () => {
|
||||
expectHttpError(got, 401, 'token_missing');
|
||||
});
|
||||
|
||||
it('passes on the error when the token could not be checked', () => {
|
||||
const lookupError = new Error('down');
|
||||
const got = runGate(requireUserActorGate(), {
|
||||
tokenAuthError: lookupError,
|
||||
});
|
||||
expect(got).toBe(lookupError);
|
||||
});
|
||||
|
||||
it('rejects a FULL-ACCESS access token too — account routes stay closed', () => {
|
||||
// The account wall is actor-type based: even a full-access PAT (which
|
||||
// the resource wall lets through) is rejected here, so it can never
|
||||
|
||||
@@ -50,7 +50,7 @@ import { assertVerifiedEmail } from '../verifiedEmail';
|
||||
// Make sure the `Express.Request.actor` augmentation is in scope.
|
||||
import '../expressAugmentation';
|
||||
|
||||
const rejectAuth = (req: Request): HttpError => {
|
||||
const rejectAuth = (req: Request): unknown => {
|
||||
if (req.requiresReauth) {
|
||||
return new HttpError(401, 'Re-authentication required', {
|
||||
legacyCode: 'reauth_required',
|
||||
@@ -71,6 +71,9 @@ const rejectAuth = (req: Request): HttpError => {
|
||||
legacyCode: 'token_auth_failed',
|
||||
});
|
||||
}
|
||||
// The token couldn't be checked, so a 401 would wrongly tell the client
|
||||
// to discard it; the error handler answers (503 for a database outage).
|
||||
if (req.tokenAuthError !== undefined) return req.tokenAuthError;
|
||||
return new HttpError(401, 'Missing authentication token', {
|
||||
legacyCode: 'token_missing',
|
||||
});
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
*/
|
||||
|
||||
import type { Request } from 'express';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { makeActor } from '../../actor';
|
||||
import {
|
||||
@@ -1132,14 +1132,150 @@ describe('resolvePublicHostedIdentity', () => {
|
||||
const reqOf = (init: {
|
||||
cookies?: Record<string, string>;
|
||||
actor?: unknown;
|
||||
headers?: Record<string, string>;
|
||||
query?: Record<string, string>;
|
||||
}): Request =>
|
||||
({
|
||||
cookies: init.cookies ?? {},
|
||||
actor: init.actor,
|
||||
headers: {},
|
||||
query: {},
|
||||
headers: init.headers ?? {},
|
||||
query: init.query ?? {},
|
||||
}) as unknown as Request;
|
||||
|
||||
describe('with the launch token in the Referer', () => {
|
||||
const host = 'launch.site.puter.localhost';
|
||||
const cookieName = HOSTED_ASSET_COOKIES.public.name;
|
||||
|
||||
const launch = async () => {
|
||||
const user = await makeUser();
|
||||
const sub = `launch-${uuidv4().slice(0, 8)}`;
|
||||
const app = await server.stores.app.create(
|
||||
{ name: sub, title: sub, index_url: `http://${host}/` },
|
||||
{ ownerUserId: user.id },
|
||||
);
|
||||
const token = await authService.getUserAppToken(
|
||||
{ user } as unknown as Parameters<
|
||||
typeof authService.getUserAppToken
|
||||
>[0],
|
||||
app.uid,
|
||||
);
|
||||
const actor = (await authService.authenticateFromToken(token))!;
|
||||
const cookie = await authService.createHostedAssetToken('public', {
|
||||
appUid: app.uid,
|
||||
userUid: user.uuid,
|
||||
sessionUuid: actor.session?.uid,
|
||||
subdomain: 'launch',
|
||||
host,
|
||||
});
|
||||
return { user, appUid: app.uid, token, cookie, actor };
|
||||
};
|
||||
|
||||
const resolve = (
|
||||
appUid: string,
|
||||
init: Parameters<typeof reqOf>[0],
|
||||
expectedHost = host,
|
||||
) =>
|
||||
resolvePublicHostedIdentity({
|
||||
req: reqOf(init),
|
||||
authService,
|
||||
sessionCookieName: 'puter_auth_token',
|
||||
expectedAppUid: appUid,
|
||||
expectedSubdomain: 'launch',
|
||||
expectedHost,
|
||||
});
|
||||
|
||||
const refererFor = (token: string) =>
|
||||
`http://${host}/?puter.auth.token=${encodeURIComponent(token)}`;
|
||||
|
||||
it('takes the cookie when the token names its user and session', async () => {
|
||||
const { user, appUid, token, cookie } = await launch();
|
||||
const authenticate = vi.spyOn(authService, 'authenticate');
|
||||
try {
|
||||
const out = await resolve(appUid, {
|
||||
cookies: { [cookieName]: cookie },
|
||||
headers: { referer: refererFor(token) },
|
||||
});
|
||||
expect(out).toMatchObject({
|
||||
source: 'public-cookie',
|
||||
userUid: user.uuid,
|
||||
hasValidPublicCookie: true,
|
||||
});
|
||||
expect(authenticate).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
authenticate.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('authenticates a query token even when it names the cookie identity', async () => {
|
||||
const { user, appUid, token, cookie } = await launch();
|
||||
const out = await resolve(appUid, {
|
||||
cookies: { [cookieName]: cookie },
|
||||
query: { 'puter.auth.token': token },
|
||||
});
|
||||
expect(out.source).toBe('query');
|
||||
expect(out.userUid).toBe(user.uuid);
|
||||
expect(
|
||||
(out as { hasValidPublicCookie?: boolean })
|
||||
.hasValidPublicCookie,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('authenticates the token when the cookie was minted for another host', async () => {
|
||||
const { user, appUid, token, cookie } = await launch();
|
||||
const out = await resolve(
|
||||
appUid,
|
||||
{
|
||||
cookies: { [cookieName]: cookie },
|
||||
headers: { referer: refererFor(token) },
|
||||
},
|
||||
'other.site.puter.localhost',
|
||||
);
|
||||
expect(out.source).toBe('referrer');
|
||||
expect(out.userUid).toBe(user.uuid);
|
||||
});
|
||||
|
||||
it('authenticates a token that names another app under the same session', async () => {
|
||||
const { user, appUid, cookie, actor } = await launch();
|
||||
const otherAppToken = server.services.token.sign('auth', {
|
||||
type: 'app-under-user',
|
||||
version: '2',
|
||||
user_uid: user.uuid,
|
||||
app_uid: `app-${uuidv4()}`,
|
||||
session_uid: actor.session!.uid,
|
||||
});
|
||||
const authenticate = vi.spyOn(authService, 'authenticate');
|
||||
try {
|
||||
const out = await resolve(appUid, {
|
||||
cookies: { [cookieName]: cookie },
|
||||
headers: { referer: refererFor(otherAppToken) },
|
||||
});
|
||||
expect(authenticate).toHaveBeenCalledTimes(1);
|
||||
expect(out.source).toBe('public-cookie');
|
||||
} finally {
|
||||
authenticate.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('never resolves a scoped access token, with or without the cookie', async () => {
|
||||
const { user, appUid, cookie } = await launch();
|
||||
const scoped = await authService.createAccessToken(
|
||||
makeActor({
|
||||
user: { id: user.id, uuid: user.uuid, username: user.username },
|
||||
}),
|
||||
[[`user:${user.uuid}:email:read`]],
|
||||
);
|
||||
const withCookie = await resolve(appUid, {
|
||||
cookies: { [cookieName]: cookie },
|
||||
headers: { referer: refererFor(scoped) },
|
||||
});
|
||||
expect(withCookie.source).toBe('public-cookie');
|
||||
const withoutCookie = await resolve(appUid, {
|
||||
headers: { referer: refererFor(scoped) },
|
||||
});
|
||||
expect(withoutCookie.source).toBe('none');
|
||||
});
|
||||
});
|
||||
|
||||
it('returns the cookie identity when present and valid', async () => {
|
||||
const user = await makeUser();
|
||||
const appUid = `app-${uuidv4()}`;
|
||||
|
||||
@@ -86,6 +86,10 @@ export interface PrivateIdentity {
|
||||
hasValidPrivateCookie?: boolean;
|
||||
}
|
||||
|
||||
type PublicCookieClaims = Awaited<
|
||||
ReturnType<AuthService['verifyHostedAssetToken']>
|
||||
>;
|
||||
|
||||
interface SubdomainLike {
|
||||
user_id?: number | null;
|
||||
}
|
||||
@@ -533,8 +537,63 @@ export async function resolvePublicHostedIdentity(opts: {
|
||||
}
|
||||
}
|
||||
|
||||
const { name: publicCookieNameV2, legacyName: publicCookieNameLegacy } =
|
||||
HOSTED_ASSET_COOKIES.public;
|
||||
const publicCookieToken =
|
||||
(typeof cookies?.[publicCookieNameV2] === 'string'
|
||||
? cookies[publicCookieNameV2]
|
||||
: null) ??
|
||||
(typeof cookies?.[publicCookieNameLegacy] === 'string'
|
||||
? cookies[publicCookieNameLegacy]
|
||||
: null);
|
||||
let publicCookieClaims: Promise<PublicCookieClaims | null> | undefined;
|
||||
const verifyPublicCookie = () =>
|
||||
(publicCookieClaims ??= (async () => {
|
||||
if (!publicCookieToken) return null;
|
||||
try {
|
||||
return await authService.verifyHostedAssetToken(
|
||||
'public',
|
||||
publicCookieToken,
|
||||
{
|
||||
appUid: expectedAppUid,
|
||||
subdomain: expectedSubdomain,
|
||||
host: expectedHost,
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
// v1-signed cookies land here and get re-minted.
|
||||
return null;
|
||||
}
|
||||
})());
|
||||
const publicCookieIdentity = (claims: PublicCookieClaims) => ({
|
||||
source: 'public-cookie' as const,
|
||||
userUid: claims.userUid,
|
||||
sessionUuid: claims.sessionUuid,
|
||||
hasValidPublicCookie: true,
|
||||
});
|
||||
|
||||
const bootstrap = getBootstrapToken(req);
|
||||
if (bootstrap) {
|
||||
// A launched app's own assets carry its launch token in the Referer.
|
||||
// When that token names the cookie's user and session, the cookie
|
||||
// already stands for it and nothing needs re-authenticating or minting.
|
||||
if (
|
||||
bootstrap.source === 'referrer' &&
|
||||
publicCookieToken &&
|
||||
expectedAppUid
|
||||
) {
|
||||
const named = authService.signedSessionClaims(bootstrap.token);
|
||||
if (named && (!named.appUid || named.appUid === expectedAppUid)) {
|
||||
const claims = await verifyPublicCookie();
|
||||
if (
|
||||
claims &&
|
||||
claims.userUid === named.userUid &&
|
||||
claims.sessionUuid === named.sessionUuid
|
||||
) {
|
||||
return publicCookieIdentity(claims);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
const actor = await authService.authenticateFromToken(
|
||||
bootstrap.token,
|
||||
@@ -554,36 +613,8 @@ export async function resolvePublicHostedIdentity(opts: {
|
||||
}
|
||||
}
|
||||
|
||||
const { name: publicCookieNameV2, legacyName: publicCookieNameLegacy } =
|
||||
HOSTED_ASSET_COOKIES.public;
|
||||
const publicCookieToken =
|
||||
(typeof cookies?.[publicCookieNameV2] === 'string'
|
||||
? cookies[publicCookieNameV2]
|
||||
: null) ??
|
||||
(typeof cookies?.[publicCookieNameLegacy] === 'string'
|
||||
? cookies[publicCookieNameLegacy]
|
||||
: null);
|
||||
if (publicCookieToken) {
|
||||
try {
|
||||
const claims = await authService.verifyHostedAssetToken(
|
||||
'public',
|
||||
publicCookieToken,
|
||||
{
|
||||
appUid: expectedAppUid,
|
||||
subdomain: expectedSubdomain,
|
||||
host: expectedHost,
|
||||
},
|
||||
);
|
||||
return {
|
||||
source: 'public-cookie',
|
||||
userUid: claims.userUid,
|
||||
sessionUuid: claims.sessionUuid,
|
||||
hasValidPublicCookie: true,
|
||||
};
|
||||
} catch {
|
||||
/* fall through — v1-signed cookies land here and get re-minted */
|
||||
}
|
||||
}
|
||||
const claims = await verifyPublicCookie();
|
||||
if (claims) return publicCookieIdentity(claims);
|
||||
|
||||
return { source: 'none' };
|
||||
}
|
||||
|
||||
@@ -2763,6 +2763,321 @@ describe('public hosted asset billing identity', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('public hosted assets requested with the launch token in the Referer', () => {
|
||||
const cookieName = HOSTED_ASSET_COOKIES.public.name;
|
||||
|
||||
const makeLaunchedApp = async () => {
|
||||
const owner = await makeUserWithHome();
|
||||
const home = (await server.stores.fsEntry.getEntryByPath(
|
||||
`/${owner.username}`,
|
||||
))!;
|
||||
const sub = `ref-${uuidv4().slice(0, 8)}`;
|
||||
const host = `${sub}.site.puter.localhost`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: sub,
|
||||
rootDirId: home.id,
|
||||
});
|
||||
await writeFile(
|
||||
owner.id,
|
||||
`/${owner.username}/index.html`,
|
||||
Buffer.from('<html></html>'),
|
||||
);
|
||||
await writeFile(
|
||||
owner.id,
|
||||
`/${owner.username}/app.css`,
|
||||
Buffer.from('body{}'),
|
||||
);
|
||||
const app = await server.stores.app.create(
|
||||
{ name: sub, title: sub, index_url: `http://${host}/` },
|
||||
{ ownerUserId: owner.id },
|
||||
);
|
||||
return { owner, sub, host, appUid: app.uid };
|
||||
};
|
||||
|
||||
const cookieFor = async (
|
||||
token: string,
|
||||
site: { sub: string; host: string; appUid: string },
|
||||
) => {
|
||||
const actor = (await server.services.auth.authenticateFromToken(
|
||||
token,
|
||||
))!;
|
||||
return server.services.auth.createHostedAssetToken('public', {
|
||||
appUid: site.appUid,
|
||||
userUid: actor.user.uuid,
|
||||
sessionUuid: actor.session?.uid,
|
||||
subdomain: site.sub,
|
||||
host: site.host,
|
||||
});
|
||||
};
|
||||
|
||||
const refererFor = (host: string, token: string) =>
|
||||
`http://${host}/?puter.app_instance_id=x&puter.auth.token=${encodeURIComponent(token)}`;
|
||||
|
||||
const serve = async (
|
||||
host: string,
|
||||
init: {
|
||||
path?: string;
|
||||
cookies?: Record<string, string>;
|
||||
headers?: Record<string, string>;
|
||||
query?: Record<string, string>;
|
||||
} = {},
|
||||
) => {
|
||||
const req = makeReq({ hostname: host, path: '/app.css', ...init });
|
||||
const { res, out } = makeRes();
|
||||
await buildMiddleware()(req, res, vi.fn());
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(out.statusCode).toBe(200);
|
||||
return { req, out };
|
||||
};
|
||||
|
||||
const claimsOf = (cookie: string, appUid: string, host: string) =>
|
||||
server.services.auth.verifyHostedAssetToken('public', cookie, {
|
||||
appUid,
|
||||
host,
|
||||
});
|
||||
|
||||
const REDIS_METHODS = [
|
||||
'get',
|
||||
'mget',
|
||||
'exists',
|
||||
'hget',
|
||||
'hgetall',
|
||||
'smembers',
|
||||
'set',
|
||||
'del',
|
||||
'expire',
|
||||
'incr',
|
||||
'incrby',
|
||||
'hset',
|
||||
'sadd',
|
||||
'pipeline',
|
||||
'multi',
|
||||
];
|
||||
|
||||
/** Redis calls, JWT signs and full authentications made by `fn`. */
|
||||
const measure = async <T>(fn: () => Promise<T>) => {
|
||||
const redis = server.clients.redis as unknown as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const redisSpies = REDIS_METHODS.filter(
|
||||
(m) => typeof redis[m] === 'function',
|
||||
).map((m) => [m, vi.spyOn(redis as never, m as never)] as const);
|
||||
const sign = vi.spyOn(server.services.token, 'sign');
|
||||
const authenticate = vi.spyOn(server.services.auth, 'authenticate');
|
||||
try {
|
||||
const result = await fn();
|
||||
const redisCalls: Record<string, number> = {};
|
||||
for (const [m, spy] of redisSpies) {
|
||||
const n = (spy as unknown as { mock: { calls: unknown[] } })
|
||||
.mock.calls.length;
|
||||
if (n) redisCalls[m] = n;
|
||||
}
|
||||
return {
|
||||
result,
|
||||
redisCalls,
|
||||
signs: sign.mock.calls.length,
|
||||
authentications: authenticate.mock.calls.length,
|
||||
};
|
||||
} finally {
|
||||
for (const [, spy] of redisSpies) spy.mockRestore();
|
||||
sign.mockRestore();
|
||||
authenticate.mockRestore();
|
||||
}
|
||||
};
|
||||
|
||||
it('costs the same as the cookie alone when the token names the cookie user and session', async () => {
|
||||
const site = await makeLaunchedApp();
|
||||
const visitor = await makeUser();
|
||||
const token = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
site.appUid,
|
||||
);
|
||||
const cookies = { [cookieName]: await cookieFor(token, site) };
|
||||
const headers = { referer: refererFor(site.host, token) };
|
||||
|
||||
// Warm the caches both paths read.
|
||||
await serve(site.host, { cookies });
|
||||
await serve(site.host, { cookies, headers });
|
||||
|
||||
const cookieOnly = await measure(() => serve(site.host, { cookies }));
|
||||
const withReferer = await measure(() =>
|
||||
serve(site.host, { cookies, headers }),
|
||||
);
|
||||
|
||||
expect(withReferer.result.req.egressActor?.user?.uuid).toBe(
|
||||
visitor.uuid,
|
||||
);
|
||||
expect(withReferer.result.out.cookies).toEqual({});
|
||||
expect(withReferer.signs).toBe(0);
|
||||
expect(withReferer.authentications).toBe(0);
|
||||
expect(withReferer.redisCalls).toEqual(cookieOnly.redisCalls);
|
||||
expect(cookieOnly.signs).toBe(0);
|
||||
});
|
||||
|
||||
it('bills the token account when the launch could not replace an older cookie', async () => {
|
||||
const site = await makeLaunchedApp();
|
||||
const previous = await makeUser();
|
||||
const current = await makeUser();
|
||||
const previousToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(previous),
|
||||
site.appUid,
|
||||
);
|
||||
const currentToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(current),
|
||||
site.appUid,
|
||||
);
|
||||
const oldCookies = {
|
||||
[cookieName]: await cookieFor(previousToken, site),
|
||||
};
|
||||
|
||||
const mint = vi
|
||||
.spyOn(server.services.auth, 'createHostedAssetToken')
|
||||
.mockRejectedValueOnce(new Error('mint failed'));
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
let launch;
|
||||
try {
|
||||
launch = await serve(site.host, {
|
||||
path: '/index.html',
|
||||
cookies: oldCookies,
|
||||
query: { 'puter.auth.token': currentToken },
|
||||
});
|
||||
} finally {
|
||||
mint.mockRestore();
|
||||
warn.mockRestore();
|
||||
}
|
||||
expect(launch.out.cookies).toEqual({});
|
||||
expect(launch.req.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
|
||||
const asset = await serve(site.host, {
|
||||
cookies: oldCookies,
|
||||
headers: { referer: refererFor(site.host, currentToken) },
|
||||
});
|
||||
expect(asset.req.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
const replaced = asset.out.cookies[cookieName];
|
||||
expect(replaced).toBeTruthy();
|
||||
expect((await claimsOf(replaced, site.appUid, site.host)).userUid).toBe(
|
||||
current.uuid,
|
||||
);
|
||||
});
|
||||
|
||||
it('bills the token account when a cached page of another account loads assets', async () => {
|
||||
// The document never reaches the server (served from an app's own
|
||||
// cache), so only the assets show which account launched it.
|
||||
const site = await makeLaunchedApp();
|
||||
const previous = await makeUser();
|
||||
const current = await makeUser();
|
||||
const previousToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(previous),
|
||||
site.appUid,
|
||||
);
|
||||
const currentToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(current),
|
||||
site.appUid,
|
||||
);
|
||||
const oldCookies = {
|
||||
[cookieName]: await cookieFor(previousToken, site),
|
||||
};
|
||||
|
||||
for (const path of ['/app.css', '/index.html']) {
|
||||
const asset = await serve(site.host, {
|
||||
path,
|
||||
cookies: oldCookies,
|
||||
headers: { referer: refererFor(site.host, currentToken) },
|
||||
});
|
||||
expect(asset.req.egressActor?.user?.uuid).toBe(current.uuid);
|
||||
const replaced = asset.out.cookies[cookieName];
|
||||
expect(
|
||||
(await claimsOf(replaced, site.appUid, site.host)).userUid,
|
||||
).toBe(current.uuid);
|
||||
}
|
||||
});
|
||||
|
||||
it('re-mints the cookie when the token names the same user under another session', async () => {
|
||||
const site = await makeLaunchedApp();
|
||||
const visitor = await makeUser();
|
||||
const appToken = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
site.appUid,
|
||||
);
|
||||
const cookies = { [cookieName]: await cookieFor(appToken, site) };
|
||||
const { session, token: sessionToken } =
|
||||
await server.services.auth.createSessionToken(visitor as never);
|
||||
|
||||
const asset = await serve(site.host, {
|
||||
cookies,
|
||||
headers: { referer: refererFor(site.host, sessionToken) },
|
||||
});
|
||||
expect(asset.req.egressActor?.user?.uuid).toBe(visitor.uuid);
|
||||
const replaced = asset.out.cookies[cookieName];
|
||||
expect(replaced).toBeTruthy();
|
||||
expect(
|
||||
(await claimsOf(replaced, site.appUid, site.host)).sessionUuid,
|
||||
).toBe(session.uuid);
|
||||
});
|
||||
|
||||
it('falls back to the cookie when the Referer token does not verify', async () => {
|
||||
const site = await makeLaunchedApp();
|
||||
const visitor = await makeUser();
|
||||
const token = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
site.appUid,
|
||||
);
|
||||
const cookies = { [cookieName]: await cookieFor(token, site) };
|
||||
const tampered = `${token.slice(0, -4)}AAAA`;
|
||||
|
||||
const asset = await serve(site.host, {
|
||||
cookies,
|
||||
headers: { referer: refererFor(site.host, tampered) },
|
||||
});
|
||||
expect(asset.req.egressActor?.user?.uuid).toBe(visitor.uuid);
|
||||
expect(asset.out.cookies).toEqual({});
|
||||
});
|
||||
|
||||
it('bills the site owner once the shared session is revoked', async () => {
|
||||
const site = await makeLaunchedApp();
|
||||
const visitor = await makeUser();
|
||||
const token = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
site.appUid,
|
||||
);
|
||||
const cookies = { [cookieName]: await cookieFor(token, site) };
|
||||
await server.services.auth.revokeAllSessionsForUserId(visitor.id);
|
||||
|
||||
const asset = await serve(site.host, {
|
||||
cookies,
|
||||
headers: { referer: refererFor(site.host, token) },
|
||||
});
|
||||
expect(asset.req.egressActor?.user?.uuid).toBe(site.owner.uuid);
|
||||
expect(asset.out.cookies).toEqual({});
|
||||
});
|
||||
|
||||
it('keeps responses that set the cookie out of shared caches', async () => {
|
||||
const site = await makeLaunchedApp();
|
||||
const visitor = await makeUser();
|
||||
const token = await server.services.auth.getUserAppToken(
|
||||
actorFor(visitor),
|
||||
site.appUid,
|
||||
);
|
||||
|
||||
const launch = await serve(site.host, {
|
||||
path: '/index.html',
|
||||
query: { 'puter.auth.token': token },
|
||||
});
|
||||
const cookie = launch.out.cookies[cookieName];
|
||||
expect(cookie).toBeTruthy();
|
||||
expect(launch.out.headers['Cache-Control']).toBe('private');
|
||||
|
||||
const asset = await serve(site.host, {
|
||||
cookies: { [cookieName]: cookie },
|
||||
headers: { referer: refererFor(site.host, token) },
|
||||
});
|
||||
expect(asset.out.cookies).toEqual({});
|
||||
expect(asset.out.headers['Cache-Control']).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('site rooted in the owner tree, root moved to Trash', () => {
|
||||
it('stops serving a trashed folder and resumes when restored', async () => {
|
||||
const owner = await makeUserWithHome();
|
||||
|
||||
@@ -548,6 +548,8 @@ export const createPuterSiteMiddleware = (
|
||||
requestHostname: host,
|
||||
}),
|
||||
);
|
||||
// A shared cache must not hand this visitor's cookie to others.
|
||||
res.set('Cache-Control', 'private');
|
||||
}
|
||||
} catch (e) {
|
||||
// Best-effort — don't block the public file serve.
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import { withSpan } from '../../../util/span.js';
|
||||
import { Context } from '../../context.js';
|
||||
import { CallSlot } from '../callSlot.js';
|
||||
import { HttpError } from '../HttpError.js';
|
||||
import { subscriptionOverride } from '../../../services/metering/consts.js';
|
||||
|
||||
@@ -412,8 +414,8 @@ async function peekKv(kv, key, limit) {
|
||||
//
|
||||
// Lifecycle is the wedge between rate and concurrent limits: rate just
|
||||
// records a tick, concurrent has to track "still in flight" → "done"
|
||||
// across a request boundary. The route middleware hooks `res.finish` /
|
||||
// `res.close`; the driver helper wraps the invocation in `try/finally`.
|
||||
// across a request boundary. The route middleware and the driver controller
|
||||
// both hold the slot in a `CallSlot`.
|
||||
|
||||
// Orphan TTL safety nets — used when the process dies between acquire
|
||||
// and release. Memory backend doesn't need one (the process is also
|
||||
@@ -1137,8 +1139,9 @@ async function resolveSubscriptionLimit(req, opts) {
|
||||
* bySubscription: { user_free: 2, unlimited: 50 } } } { concurrent: { limit:
|
||||
* 10, backend: 'redis', scope: 'expensive-op' } }
|
||||
*
|
||||
* On accept, schedules release on `res.finish` / `res.close` so even aborted
|
||||
* requests give their slot back. On reject, 429 with the same
|
||||
* On accept, holds the slot until the response is done and any work the handler
|
||||
* registered through `keepConcurrencySlot` has settled, so work that outlives a
|
||||
* caller who hung up still counts. On reject, 429 with the same
|
||||
* `too_many_requests` legacyCode as the rate gate (clients already handle that
|
||||
* branch). Fails open on backend error.
|
||||
*/
|
||||
@@ -1164,13 +1167,11 @@ export function concurrencyGate(opts) {
|
||||
return;
|
||||
}
|
||||
|
||||
// `finish` (response sent) and `close` (connection closed, possibly
|
||||
// aborted before finish) can both fire; the guard releases once.
|
||||
const { release } = guardSlot(slot, '[concurrent]');
|
||||
const onDone = () => void release();
|
||||
res.once('finish', onDone);
|
||||
res.once('close', onDone);
|
||||
if (res.destroyed || res.writableFinished) onDone();
|
||||
const held = new CallSlot();
|
||||
held.attach(guardSlot(slot, '[concurrent]'), res);
|
||||
if (Context.current()) {
|
||||
Context.set('keepConcurrencySlot', (work) => held.keep(work));
|
||||
}
|
||||
next();
|
||||
};
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { Context, runWithContext } from '../../context.js';
|
||||
import { isHttpError } from '../HttpError.js';
|
||||
import { setupTestServer } from '../../../testUtil.ts';
|
||||
import {
|
||||
@@ -977,6 +978,27 @@ describe('concurrencyGate — memory backend', () => {
|
||||
expect(isHttpError(third.err)).toBe(true);
|
||||
});
|
||||
|
||||
it('holds the slot past the response for work the handler keeps', async () => {
|
||||
const opts = { limit: 1, key: 'ip', scope: 'cg-kept-work' };
|
||||
let finishWork;
|
||||
const work = new Promise((resolve) => {
|
||||
finishWork = resolve;
|
||||
});
|
||||
const res = makeRes();
|
||||
await runWithContext({}, async () => {
|
||||
await concurrencyGate(opts)(baseReq(), res, () => {
|
||||
Context.get('keepConcurrencySlot')(work);
|
||||
});
|
||||
});
|
||||
res.emit('close');
|
||||
await new Promise((r) => setImmediate(r));
|
||||
expect((await runGate(opts, baseReq())).err?.statusCode).toBe(429);
|
||||
|
||||
finishWork();
|
||||
await new Promise((r) => setImmediate(r));
|
||||
expect((await runGate(opts, baseReq())).err).toBeUndefined();
|
||||
});
|
||||
|
||||
it('different keys do not share slots', async () => {
|
||||
const opts = { limit: 1, key: 'ip', scope: 'cg-keyed' };
|
||||
const a = await runGate(opts, baseReq({ ip: '1.1.1.1' }));
|
||||
|
||||
@@ -263,10 +263,10 @@ export interface RouteOptions {
|
||||
rateLimit?: RouteRateLimit | RouteRateLimit[];
|
||||
|
||||
/**
|
||||
* Cap simultaneous in-flight requests per key; slots are released on
|
||||
* response finish or close. `bySubscription` overrides `limit` per
|
||||
* `SubscriptionPolicy.id` when metering is wired in. `key`, `scope`,
|
||||
* `backend` as in `rateLimit`.
|
||||
* Cap simultaneous in-flight requests per key; a slot is released once the
|
||||
* response is done and any work the handler kept it for has settled.
|
||||
* `bySubscription` overrides `limit` per `SubscriptionPolicy.id` when
|
||||
* metering is wired in. `key`, `scope`, `backend` as in `rateLimit`.
|
||||
*/
|
||||
concurrent?: {
|
||||
limit: number;
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
messagesHaveCompaction,
|
||||
toAnthropicContextManagement,
|
||||
toOpenAiContextManagement,
|
||||
wantsCompaction,
|
||||
} from './compaction.js';
|
||||
|
||||
describe('toOpenAiContextManagement', () => {
|
||||
it('returns undefined when compaction is off', () => {
|
||||
expect(toOpenAiContextManagement({})).toBeUndefined();
|
||||
expect(
|
||||
toOpenAiContextManagement({ compaction: false }),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('builds a compaction entry from `true`', () => {
|
||||
expect(toOpenAiContextManagement({ compaction: true })).toEqual([
|
||||
{ type: 'compaction' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('maps trigger_tokens to compact_threshold', () => {
|
||||
expect(
|
||||
toOpenAiContextManagement({ compaction: { trigger_tokens: 1000 } }),
|
||||
).toEqual([{ type: 'compaction', compact_threshold: 1000 }]);
|
||||
});
|
||||
|
||||
it('passes a raw context_management payload through verbatim', () => {
|
||||
const raw = [{ type: 'compaction', compact_threshold: 5 }];
|
||||
expect(
|
||||
toOpenAiContextManagement({
|
||||
compaction: true,
|
||||
context_management: raw,
|
||||
}),
|
||||
).toBe(raw);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toAnthropicContextManagement', () => {
|
||||
it('returns undefined when compaction is off', () => {
|
||||
expect(toAnthropicContextManagement({})).toBeUndefined();
|
||||
});
|
||||
|
||||
it('builds a compact_20260112 edit from `true`', () => {
|
||||
expect(toAnthropicContextManagement({ compaction: true })).toEqual({
|
||||
edits: [{ type: 'compact_20260112' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('maps trigger_tokens to an input_tokens trigger', () => {
|
||||
expect(
|
||||
toAnthropicContextManagement({
|
||||
compaction: { trigger_tokens: 2000 },
|
||||
}),
|
||||
).toEqual({
|
||||
edits: [
|
||||
{
|
||||
type: 'compact_20260112',
|
||||
trigger: { type: 'input_tokens', value: 2000 },
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('passes a raw context_management payload through verbatim', () => {
|
||||
const raw = { edits: [{ type: 'compact_20260112' }] };
|
||||
expect(
|
||||
toAnthropicContextManagement({ context_management: raw }),
|
||||
).toBe(raw);
|
||||
});
|
||||
});
|
||||
|
||||
describe('wantsCompaction', () => {
|
||||
it('is false without opt-in', () => {
|
||||
expect(wantsCompaction({})).toBe(false);
|
||||
expect(wantsCompaction({ compaction: false })).toBe(false);
|
||||
});
|
||||
|
||||
it('is true for the neutral opt-in or a raw passthrough', () => {
|
||||
expect(wantsCompaction({ compaction: true })).toBe(true);
|
||||
expect(wantsCompaction({ compaction: { trigger_tokens: 1 } })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(wantsCompaction({ context_management: [] })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('messagesHaveCompaction', () => {
|
||||
it('detects a round-tripped compaction content block', () => {
|
||||
expect(
|
||||
messagesHaveCompaction([
|
||||
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'compaction', encrypted_content: 'ENC' },
|
||||
],
|
||||
},
|
||||
]),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('is false for ordinary messages or non-arrays', () => {
|
||||
expect(
|
||||
messagesHaveCompaction([
|
||||
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
|
||||
]),
|
||||
).toBe(false);
|
||||
expect(messagesHaveCompaction(undefined)).toBe(false);
|
||||
expect(messagesHaveCompaction('nope')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,718 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import {
|
||||
DetectDocumentTextCommand,
|
||||
InvalidS3ObjectException,
|
||||
TextractClient,
|
||||
UnsupportedDocumentException,
|
||||
} from '@aws-sdk/client-textract';
|
||||
import { Mistral } from '@mistralai/mistralai';
|
||||
import { Actor } from '../../core/actor.js';
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import {
|
||||
awsCredentialsFrom,
|
||||
type AwsCredentials,
|
||||
} from '../../util/awsCredentials.js';
|
||||
import { mimeFromName } from '../../util/fileSigning.js';
|
||||
import {
|
||||
assertActorHasCredits,
|
||||
insufficientCreditsError,
|
||||
} from '../../services/metering/enforcement.js';
|
||||
import type { CreditHold } from '../../services/metering/types.js';
|
||||
import { PuterDriver } from '../types.js';
|
||||
import {
|
||||
type AiMeteringService,
|
||||
withAiCostFactor,
|
||||
} from '../util/aiCostFactor.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js';
|
||||
import {
|
||||
type LoadedFile,
|
||||
type ResolvedFileInput,
|
||||
resolveFileInput,
|
||||
} from '../util/fileInput.js';
|
||||
import {
|
||||
type ProviderCatalog,
|
||||
ProviderRegistry,
|
||||
readProviderKey,
|
||||
} from '../util/providerRegistry.js';
|
||||
import { OCR_COSTS } from './costs.js';
|
||||
import {
|
||||
DEFAULT_OCR_MODEL,
|
||||
findOcrModel,
|
||||
OCR_MAX_INPUT_BYTES,
|
||||
OCR_MAX_PAGES,
|
||||
RETIRED_OCR_MODELS,
|
||||
type OcrModel,
|
||||
type OcrProviderId,
|
||||
} from './models.js';
|
||||
import { countPdfPages } from './pdfPages.js';
|
||||
import { textractRates } from './textractPricing.js';
|
||||
|
||||
/**
|
||||
* Driver implementing `puter-ocr` — document OCR. Two providers: •
|
||||
* `aws-textract` — AWS Textract (region-aware clients; direct S3 source when
|
||||
* available) • `mistral` — Mistral OCR (URL/data-URL based)
|
||||
*/
|
||||
interface RecognizeArgs {
|
||||
source?: unknown;
|
||||
file?: unknown;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
// Mistral-specific options — ignored by Textract.
|
||||
pages?: number[];
|
||||
includeImageBase64?: boolean;
|
||||
imageLimit?: number;
|
||||
imageMinSize?: number;
|
||||
bboxAnnotationFormat?: unknown;
|
||||
documentAnnotationFormat?: unknown;
|
||||
documentAnnotationPrompt?: string;
|
||||
tableFormat?: string;
|
||||
extractHeader?: boolean;
|
||||
extractFooter?: boolean;
|
||||
test_mode?: boolean;
|
||||
}
|
||||
|
||||
/** One recognized line, in the same shape for every provider. */
|
||||
interface OcrBlock {
|
||||
type: string;
|
||||
text: string;
|
||||
confidence?: number;
|
||||
page: number;
|
||||
}
|
||||
|
||||
interface TextractBlock {
|
||||
BlockType?: string;
|
||||
Confidence?: number;
|
||||
Text?: string;
|
||||
}
|
||||
|
||||
interface MistralOcrResponse {
|
||||
model?: string;
|
||||
pages?: Array<{
|
||||
index?: number;
|
||||
markdown?: string;
|
||||
images?: unknown[];
|
||||
dimensions?: unknown;
|
||||
}>;
|
||||
documentAnnotation?: string | null;
|
||||
usageInfo?: { pagesProcessed?: number };
|
||||
}
|
||||
|
||||
interface MistralOcrClient {
|
||||
ocr: {
|
||||
process: (
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<MistralOcrResponse>;
|
||||
};
|
||||
}
|
||||
|
||||
// Aliases callers may use in place of a canonical provider id. Resolved here
|
||||
// rather than in the SDK so a new alias reaches every caller at once.
|
||||
const OCR_CATALOG: ProviderCatalog = {
|
||||
label: 'OCR',
|
||||
ids: ['aws-textract', 'mistral'],
|
||||
defaultId: 'aws-textract',
|
||||
aliases: {
|
||||
aws: 'aws-textract',
|
||||
'aws-textract': 'aws-textract',
|
||||
textract: 'aws-textract',
|
||||
mistral: 'mistral',
|
||||
'mistral-ocr': 'mistral',
|
||||
},
|
||||
};
|
||||
|
||||
/** Recognizes one input with one provider's model. */
|
||||
type OcrRecognizer = (
|
||||
input: ResolvedFileInput,
|
||||
args: RecognizeArgs,
|
||||
model: OcrModel,
|
||||
actor: Actor,
|
||||
) => Promise<unknown>;
|
||||
|
||||
const NOT_CONFIGURED: Record<OcrProviderId, string> = {
|
||||
'aws-textract': 'AWS credentials not configured',
|
||||
mistral: 'Mistral OCR not configured',
|
||||
};
|
||||
|
||||
const TABLE_FORMATS = new Set(['markdown', 'html']);
|
||||
|
||||
const DEFAULT_TEXTRACT_REGION = 'us-west-2';
|
||||
const TEXTRACT_PAGE_USAGE_TYPE = 'aws-textract:detect-document-text:page';
|
||||
|
||||
const badRequest = (message: string) =>
|
||||
new HttpError(400, message, { legacyCode: 'bad_request' });
|
||||
|
||||
/**
|
||||
* Accept a Mistral annotation format in the REST spelling
|
||||
* (`json_schema.schema`) or the SDK spelling (`jsonSchema.schemaDefinition`)
|
||||
* and return the SDK shape.
|
||||
*/
|
||||
const toMistralResponseFormat = (
|
||||
name: string,
|
||||
value: unknown,
|
||||
): Record<string, unknown> => {
|
||||
const format = value as Record<string, unknown> | null;
|
||||
const jsonSchema = (format?.jsonSchema ?? format?.json_schema) as
|
||||
Record<string, unknown> | undefined;
|
||||
const schema = jsonSchema?.schemaDefinition ?? jsonSchema?.schema;
|
||||
if (
|
||||
!format ||
|
||||
typeof format !== 'object' ||
|
||||
(format.type !== undefined && format.type !== 'json_schema') ||
|
||||
!schema ||
|
||||
typeof schema !== 'object'
|
||||
) {
|
||||
throw badRequest(
|
||||
`\`${name}\` must be { type: 'json_schema', json_schema: { name, schema } }`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
type: 'json_schema',
|
||||
jsonSchema: {
|
||||
name:
|
||||
typeof jsonSchema?.name === 'string'
|
||||
? jsonSchema.name
|
||||
: 'annotation',
|
||||
schemaDefinition: schema,
|
||||
...(typeof jsonSchema?.description === 'string' && {
|
||||
description: jsonSchema.description,
|
||||
}),
|
||||
...(typeof jsonSchema?.strict === 'boolean' && {
|
||||
strict: jsonSchema.strict,
|
||||
}),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const inputMimeType = (loaded: LoadedFile): string =>
|
||||
loaded.mimeType ??
|
||||
mimeFromName(loaded.filename) ??
|
||||
'application/octet-stream';
|
||||
|
||||
/**
|
||||
* Declared documents (PDF, DOCX, PPTX, ...) and PDF bytes are documents; images
|
||||
* and untyped bytes are read as a single image.
|
||||
*/
|
||||
const isDocumentInput = (loaded: LoadedFile): boolean => {
|
||||
const mime = inputMimeType(loaded);
|
||||
return (
|
||||
loaded.buffer.subarray(0, 4).toString('latin1') === '%PDF' ||
|
||||
loaded.filename.toLowerCase().endsWith('.pdf') ||
|
||||
(!mime.startsWith('image/') && mime !== 'application/octet-stream')
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* The most pages a call can bill, known before the provider runs: a PDF's page
|
||||
* count, or a size-based guess for other documents, capped by a `pages`
|
||||
* selection and by what the provider reads in one call. Reads the input only
|
||||
* for a provider that takes more than one page.
|
||||
*/
|
||||
const estimateOcrPages = async (
|
||||
input: ResolvedFileInput,
|
||||
provider: OcrProviderId,
|
||||
selection?: unknown,
|
||||
): Promise<number> => {
|
||||
const maxPages = OCR_MAX_PAGES[provider];
|
||||
if (maxPages <= 1) return 1;
|
||||
const loaded = await input.load();
|
||||
if (!isDocumentInput(loaded)) return 1;
|
||||
// A document we can't count is billed in proportion to its size, reaching
|
||||
// the provider's page limit at its size limit (20 pages per MB for
|
||||
// Mistral). Small files stay cheap; a large one can't pass as one page.
|
||||
const documentPages =
|
||||
(await countPdfPages(loaded.buffer, maxPages)) ??
|
||||
Math.ceil(
|
||||
(loaded.buffer.length * maxPages) / OCR_MAX_INPUT_BYTES[provider],
|
||||
);
|
||||
const selected = Array.isArray(selection)
|
||||
? new Set(
|
||||
selection.filter(
|
||||
(page) => Number.isInteger(page) && (page as number) >= 0,
|
||||
),
|
||||
).size
|
||||
: 0;
|
||||
return Math.max(1, Math.min(documentPages, maxPages, selected || Infinity));
|
||||
};
|
||||
|
||||
export class OCRDriver extends PuterDriver {
|
||||
readonly driverInterface = 'puter-ocr';
|
||||
readonly driverName = 'ai-ocr';
|
||||
|
||||
// Shared AI policy — see `drivers/util/aiLimits.ts` for the tier table.
|
||||
readonly rateLimit = AI_RATE_LIMIT;
|
||||
readonly concurrent = AI_CONCURRENT;
|
||||
|
||||
/** Metering scoped to this driver. Lazy: services wire up after drivers. */
|
||||
get #aiMetering(): AiMeteringService {
|
||||
return withAiCostFactor(
|
||||
this.services.metering,
|
||||
this.clients.event,
|
||||
this.driverName,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Textract is reported at the configured region's first-tier rate. A call
|
||||
* that reads its file from a bucket in another region is billed at that
|
||||
* region's rate instead, which this per-usage-type report can't show.
|
||||
*/
|
||||
override getReportedCosts() {
|
||||
const textractRegion =
|
||||
this.#awsConfig?.region ?? DEFAULT_TEXTRACT_REGION;
|
||||
return Object.entries(OCR_COSTS).map(([usageType, ucentsPerUnit]) => ({
|
||||
usageType,
|
||||
ucentsPerUnit:
|
||||
usageType === TEXTRACT_PAGE_USAGE_TYPE
|
||||
? this.#textractRates.peek(textractRegion)
|
||||
: ucentsPerUnit,
|
||||
unit: 'page',
|
||||
source: 'driver:aiOcr',
|
||||
}));
|
||||
}
|
||||
|
||||
// Older SDK bundles name the provider in the driver slot instead of
|
||||
// passing `{ provider }`; `#resolveModel` reads the requested alias
|
||||
// back off the Context.
|
||||
readonly driverAliases = [...OCR_CATALOG.ids];
|
||||
readonly isDefault = true;
|
||||
|
||||
// Textract state — one client per region.
|
||||
#textractClients: Record<string, TextractClient> = {};
|
||||
#awsConfig: { credentials: AwsCredentials; region: string } | null = null;
|
||||
/** DetectDocumentText rates per region, from AWS's public price list. */
|
||||
#textractRates = textractRates(OCR_COSTS[TEXTRACT_PAGE_USAGE_TYPE]);
|
||||
|
||||
// Mistral state.
|
||||
#mistral: MistralOcrClient | null = null;
|
||||
|
||||
/** Configured providers, each a recognizer bound to its state above. */
|
||||
#providers = new ProviderRegistry<OcrRecognizer>(OCR_CATALOG);
|
||||
|
||||
override onServerStart() {
|
||||
const providers = this.config.providers ?? {};
|
||||
|
||||
const textract = providers['aws-textract'] as
|
||||
Record<string, unknown> | undefined;
|
||||
const textractAws = (textract?.aws ?? textract) as
|
||||
Record<string, unknown> | undefined;
|
||||
const textractCredentials = awsCredentialsFrom(textractAws);
|
||||
const textractRegion =
|
||||
(textractAws?.region as string | undefined) ??
|
||||
(textract?.region as string | undefined) ??
|
||||
DEFAULT_TEXTRACT_REGION;
|
||||
if (textractCredentials) {
|
||||
this.#awsConfig = {
|
||||
credentials: textractCredentials,
|
||||
region: textractRegion,
|
||||
};
|
||||
// Warm the configured region so the cost report has its rate.
|
||||
void this.#textractRates.rate(textractRegion);
|
||||
this.#providers.register(
|
||||
'aws-textract',
|
||||
(input, _args, model, actor) =>
|
||||
this.#textractRecognize(input, model, actor),
|
||||
);
|
||||
}
|
||||
|
||||
const mistralKey = readProviderKey(providers['mistral-ocr']);
|
||||
if (mistralKey) {
|
||||
try {
|
||||
this.#mistral = new Mistral({
|
||||
apiKey: mistralKey,
|
||||
}) as unknown as MistralOcrClient;
|
||||
this.#providers.register(
|
||||
'mistral',
|
||||
(input, args, model, actor) =>
|
||||
this.#mistralRecognize(input, args, model, actor),
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
'[OCRDriver] Failed to init Mistral:',
|
||||
(e as Error).message,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async recognize(args: RecognizeArgs) {
|
||||
if (args.test_mode) return sampleResponse();
|
||||
|
||||
const model = this.#resolveModel(args);
|
||||
|
||||
const actor = Context.get('actor');
|
||||
if (!actor)
|
||||
throw new HttpError(401, 'Authentication required', {
|
||||
legacyCode: 'unauthorized',
|
||||
});
|
||||
|
||||
const input = args.source ?? args.file;
|
||||
if (!input) throw badRequest('`source` is required');
|
||||
|
||||
const recognize = this.#providers.get(model.provider);
|
||||
if (!recognize)
|
||||
throw new HttpError(500, NOT_CONFIGURED[model.provider], {
|
||||
legacyCode: 'internal_error',
|
||||
});
|
||||
|
||||
// Cheap, cached check: reject a zero-balance actor before paying for
|
||||
// the file load and page estimate, which a real credit check would
|
||||
// reject anyway once it runs.
|
||||
await assertActorHasCredits(this.services.metering, actor, this.config);
|
||||
|
||||
// A Puter file is resolved and access-checked here but read only if
|
||||
// the provider needs its bytes; Textract reads it straight from S3.
|
||||
const resolved = await resolveFileInput(
|
||||
this.stores,
|
||||
this.services.fs,
|
||||
actor,
|
||||
input,
|
||||
{
|
||||
acceptWebInput: true,
|
||||
maxBytes: OCR_MAX_INPUT_BYTES[model.provider],
|
||||
},
|
||||
);
|
||||
|
||||
return recognize(resolved, args, model, actor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide which model handles a call. A `model` picks its own provider; an
|
||||
* explicit `provider`, then the legacy driver alias the caller dispatched
|
||||
* through, then whichever provider is configured pick that provider's
|
||||
* default model.
|
||||
*/
|
||||
#resolveModel(args: RecognizeArgs): OcrModel {
|
||||
const provider = args.provider
|
||||
? (this.#providers.resolve(args.provider) as OcrProviderId)
|
||||
: undefined;
|
||||
|
||||
if (args.model !== undefined) {
|
||||
if (typeof args.model !== 'string' || !args.model.trim())
|
||||
throw badRequest('`model` must be a non-empty string');
|
||||
const retired = RETIRED_OCR_MODELS[args.model.trim().toLowerCase()];
|
||||
if (retired)
|
||||
throw badRequest(
|
||||
`${args.model} is no longer available: ${retired}`,
|
||||
);
|
||||
const model = findOcrModel(args.model);
|
||||
if (!model) throw badRequest(`Unknown OCR model: ${args.model}`);
|
||||
if (provider && model.provider !== provider)
|
||||
throw badRequest(
|
||||
`OCR model ${args.model} is not served by provider ${args.provider}`,
|
||||
);
|
||||
return model;
|
||||
}
|
||||
|
||||
const hinted =
|
||||
provider ?? this.#providers.normalize(Context.get('driverName'));
|
||||
if (!hinted && this.#providers.names().length === 0)
|
||||
throw new HttpError(500, 'No OCR provider configured', {
|
||||
legacyCode: 'internal_error',
|
||||
});
|
||||
const chosen = (hinted ?? this.#providers.defaultId()) as OcrProviderId;
|
||||
return findOcrModel(DEFAULT_OCR_MODEL[chosen])!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse a call the balance can't cover, and hold its cost while the
|
||||
* provider runs so the account's concurrent calls see it. Usage is still
|
||||
* metered from the pages the provider reports.
|
||||
*/
|
||||
async #holdCredits(
|
||||
actor: Actor,
|
||||
usageType: string,
|
||||
cost: number,
|
||||
): Promise<CreditHold> {
|
||||
// Priced at the cost factor usage is recorded at.
|
||||
const hold = await this.#aiMetering.reserveAiCredits(
|
||||
actor,
|
||||
usageType,
|
||||
cost,
|
||||
);
|
||||
if (!hold) throw insufficientCreditsError();
|
||||
return hold;
|
||||
}
|
||||
|
||||
// -- AWS Textract -------------------------------------------------
|
||||
|
||||
#textractClientFor(region: string): TextractClient {
|
||||
const cached = this.#textractClients[region];
|
||||
if (cached) return cached;
|
||||
const client = new TextractClient({
|
||||
credentials: this.#awsConfig!.credentials,
|
||||
region,
|
||||
});
|
||||
this.#textractClients[region] = client;
|
||||
return client;
|
||||
}
|
||||
|
||||
async #textractRecognize(
|
||||
input: ResolvedFileInput,
|
||||
model: OcrModel,
|
||||
actor: Actor,
|
||||
) {
|
||||
// Prefer S3 direct source if the file is FS-backed; fall back to raw bytes.
|
||||
const { fsEntry } = input;
|
||||
const s3Info =
|
||||
fsEntry && fsEntry.bucket && fsEntry.bucketRegion
|
||||
? {
|
||||
bucket: fsEntry.bucket,
|
||||
bucketRegion: fsEntry.bucketRegion,
|
||||
key: fsEntry.uuid,
|
||||
}
|
||||
: null;
|
||||
|
||||
const configuredRegion =
|
||||
this.#awsConfig!.region ?? DEFAULT_TEXTRACT_REGION;
|
||||
|
||||
// Both regions the call may run in are priced up front, so the hold
|
||||
// covers either and a refresh mid-call can't change the charge.
|
||||
const [configuredRate, s3Rate] = await Promise.all([
|
||||
this.#textractRates.rate(configuredRegion),
|
||||
s3Info ? this.#textractRates.rate(s3Info.bucketRegion) : undefined,
|
||||
]);
|
||||
|
||||
const tryRun = async (useS3: boolean) => {
|
||||
const region =
|
||||
s3Info && useS3 ? s3Info.bucketRegion : configuredRegion;
|
||||
const client = this.#textractClientFor(region);
|
||||
const document =
|
||||
s3Info && useS3
|
||||
? { S3Object: { Bucket: s3Info.bucket, Name: s3Info.key } }
|
||||
: { Bytes: (await input.load()).buffer };
|
||||
return client.send(
|
||||
new DetectDocumentTextCommand({ Document: document }),
|
||||
);
|
||||
};
|
||||
|
||||
const hold = await this.#holdCredits(
|
||||
actor,
|
||||
model.pageUsageType,
|
||||
Math.max(configuredRate, s3Rate ?? 0) *
|
||||
(await estimateOcrPages(input, model.provider)),
|
||||
);
|
||||
try {
|
||||
let response;
|
||||
let costPerPage: number;
|
||||
try {
|
||||
try {
|
||||
response = await tryRun(Boolean(s3Info));
|
||||
costPerPage = s3Rate ?? configuredRate;
|
||||
} catch (err) {
|
||||
if (!(s3Info && err instanceof InvalidS3ObjectException))
|
||||
throw err;
|
||||
response = await tryRun(false);
|
||||
costPerPage = configuredRate;
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof UnsupportedDocumentException)
|
||||
throw badRequest(
|
||||
'AWS Textract reads JPEG, PNG, TIFF and single-page PDF documents; use a Mistral OCR model for multi-page PDFs and other formats',
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
|
||||
const blocks: OcrBlock[] = [];
|
||||
let pageCount = 0;
|
||||
for (const block of (response.Blocks ?? []) as TextractBlock[]) {
|
||||
if (block.BlockType === 'PAGE') {
|
||||
pageCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (block.BlockType !== 'LINE' || !block.Text) continue;
|
||||
blocks.push({
|
||||
type: 'text/textract:LINE',
|
||||
text: block.Text,
|
||||
confidence: Number(block.Confidence ?? 0),
|
||||
page: Math.max(pageCount - 1, 0),
|
||||
});
|
||||
}
|
||||
|
||||
const pages = pageCount || 1;
|
||||
this.#aiMetering.incrementUsage(
|
||||
actor,
|
||||
model.pageUsageType,
|
||||
pages,
|
||||
costPerPage * pages,
|
||||
);
|
||||
return {
|
||||
model: model.id,
|
||||
blocks,
|
||||
text: blocks.map((b) => b.text).join('\n'),
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
}
|
||||
|
||||
// -- Mistral OCR --------------------------------------------------
|
||||
|
||||
async #mistralRecognize(
|
||||
input: ResolvedFileInput,
|
||||
args: RecognizeArgs,
|
||||
model: OcrModel,
|
||||
actor: Actor,
|
||||
) {
|
||||
const loaded = await input.load();
|
||||
const payload: Record<string, unknown> = {
|
||||
model: model.id,
|
||||
document: this.#mistralBuildChunk(loaded),
|
||||
};
|
||||
if (args.pages) payload.pages = args.pages;
|
||||
if (args.includeImageBase64 !== undefined)
|
||||
payload.includeImageBase64 = args.includeImageBase64;
|
||||
if (typeof args.imageLimit === 'number')
|
||||
payload.imageLimit = args.imageLimit;
|
||||
if (typeof args.imageMinSize === 'number')
|
||||
payload.imageMinSize = args.imageMinSize;
|
||||
if (args.bboxAnnotationFormat !== undefined)
|
||||
payload.bboxAnnotationFormat = toMistralResponseFormat(
|
||||
'bboxAnnotationFormat',
|
||||
args.bboxAnnotationFormat,
|
||||
);
|
||||
if (args.documentAnnotationFormat !== undefined)
|
||||
payload.documentAnnotationFormat = toMistralResponseFormat(
|
||||
'documentAnnotationFormat',
|
||||
args.documentAnnotationFormat,
|
||||
);
|
||||
if (typeof args.documentAnnotationPrompt === 'string')
|
||||
payload.documentAnnotationPrompt = args.documentAnnotationPrompt;
|
||||
if (args.tableFormat !== undefined) {
|
||||
if (!TABLE_FORMATS.has(args.tableFormat))
|
||||
throw badRequest("`tableFormat` must be 'markdown' or 'html'");
|
||||
payload.tableFormat = args.tableFormat;
|
||||
}
|
||||
if (typeof args.extractHeader === 'boolean')
|
||||
payload.extractHeader = args.extractHeader;
|
||||
if (typeof args.extractFooter === 'boolean')
|
||||
payload.extractFooter = args.extractFooter;
|
||||
|
||||
const annotations =
|
||||
payload.documentAnnotationFormat !== undefined ||
|
||||
payload.bboxAnnotationFormat !== undefined;
|
||||
const costPerPage =
|
||||
OCR_COSTS[model.pageUsageType] +
|
||||
(annotations && model.annotationUsageType
|
||||
? OCR_COSTS[model.annotationUsageType]
|
||||
: 0);
|
||||
const hold = await this.#holdCredits(
|
||||
actor,
|
||||
model.pageUsageType,
|
||||
costPerPage *
|
||||
(await estimateOcrPages(input, model.provider, args.pages)),
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await this.#mistral!.ocr.process(payload);
|
||||
this.#recordMistralUsage(response, model, actor, annotations);
|
||||
return this.#normalizeMistralResponse(response, model);
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
}
|
||||
|
||||
#mistralBuildChunk(loaded: LoadedFile): Record<string, unknown> {
|
||||
const mime = inputMimeType(loaded);
|
||||
const dataUrl = `data:${mime};base64,${loaded.buffer.toString('base64')}`;
|
||||
if (isDocumentInput(loaded)) {
|
||||
return {
|
||||
type: 'document_url',
|
||||
documentUrl: dataUrl,
|
||||
documentName: loaded.filename,
|
||||
};
|
||||
}
|
||||
return { type: 'image_url', imageUrl: { url: dataUrl } };
|
||||
}
|
||||
|
||||
#normalizeMistralResponse(response: MistralOcrResponse, model: OcrModel) {
|
||||
const pages = response?.pages ?? [];
|
||||
const blocks: OcrBlock[] = [];
|
||||
for (const [position, page] of pages.entries()) {
|
||||
if (typeof page?.markdown !== 'string') continue;
|
||||
const lines = page.markdown
|
||||
.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
for (const line of lines) {
|
||||
blocks.push({
|
||||
type: 'text/mistral:LINE',
|
||||
text: line,
|
||||
page: page.index ?? position,
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
model: response?.model ?? model.id,
|
||||
pages,
|
||||
usage_info: response?.usageInfo,
|
||||
...(typeof response?.documentAnnotation === 'string' && {
|
||||
document_annotation: response.documentAnnotation,
|
||||
}),
|
||||
blocks,
|
||||
text: blocks.map((b) => b.text).join('\n'),
|
||||
};
|
||||
}
|
||||
|
||||
#recordMistralUsage(
|
||||
response: MistralOcrResponse,
|
||||
model: OcrModel,
|
||||
actor: Actor,
|
||||
annotations: boolean,
|
||||
) {
|
||||
try {
|
||||
const pagesProcessed =
|
||||
response?.usageInfo?.pagesProcessed ??
|
||||
(Array.isArray(response?.pages) ? response.pages.length : 1);
|
||||
this.#aiMetering.incrementUsage(
|
||||
actor,
|
||||
model.pageUsageType,
|
||||
pagesProcessed,
|
||||
OCR_COSTS[model.pageUsageType] * pagesProcessed,
|
||||
);
|
||||
if (annotations && model.annotationUsageType) {
|
||||
this.#aiMetering.incrementUsage(
|
||||
actor,
|
||||
model.annotationUsageType,
|
||||
pagesProcessed,
|
||||
OCR_COSTS[model.annotationUsageType] * pagesProcessed,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Non-critical.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sampleResponse() {
|
||||
return {
|
||||
model: 'test-mode',
|
||||
blocks: [
|
||||
{
|
||||
type: 'text/puter:sample-output',
|
||||
confidence: 1,
|
||||
text: 'test_mode is enabled; this is a sample OCR response.',
|
||||
page: 0,
|
||||
},
|
||||
],
|
||||
text: 'test_mode is enabled; this is a sample OCR response.',
|
||||
};
|
||||
}
|
||||
@@ -1,511 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { pipeline, Readable, Transform } from 'node:stream';
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import type { DriverStreamResult } from '../meta.js';
|
||||
import { OUTPUT_FORMATS } from '../ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.js';
|
||||
import { PuterDriver } from '../types.js';
|
||||
import {
|
||||
type AiMeteringService,
|
||||
withAiCostFactor,
|
||||
withStreamedAiCharge,
|
||||
} from '../util/aiCostFactor.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js';
|
||||
import { loadFileInput } from '../util/fileInput.js';
|
||||
import {
|
||||
type ProviderCatalog,
|
||||
ProviderRegistry,
|
||||
readProviderKey,
|
||||
} from '../util/providerRegistry.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../util/testMode.js';
|
||||
import { upstreamBodyStream, upstreamFetch } from '../util/upstreamErrors.js';
|
||||
import { VOICE_CHANGER_COSTS } from './costs.js';
|
||||
|
||||
/**
|
||||
* Driver implementing `puter-speech2speech` — voice changer. Currently a single
|
||||
* provider (ElevenLabs).
|
||||
*/
|
||||
|
||||
const DEFAULT_MODEL = 'eleven_multilingual_sts_v2';
|
||||
const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM';
|
||||
const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128';
|
||||
const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024;
|
||||
// Covers the upload as well as the conversion.
|
||||
const CONVERT_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
|
||||
const SUPPORTED_MODELS = Object.keys(VOICE_CHANGER_COSTS).map(
|
||||
(key) => key.split(':')[1]!,
|
||||
);
|
||||
|
||||
const VOICE_CHANGER_CATALOG: ProviderCatalog = {
|
||||
label: 'Speech-to-speech',
|
||||
ids: ['elevenlabs'],
|
||||
defaultId: 'elevenlabs',
|
||||
aliases: { elevenlabs: 'elevenlabs' },
|
||||
};
|
||||
|
||||
interface ElevenLabsConfig {
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
defaultVoiceId: string;
|
||||
defaultModelId: string;
|
||||
}
|
||||
|
||||
interface ConvertArgs {
|
||||
audio: unknown;
|
||||
provider?: string;
|
||||
voice?: string;
|
||||
voice_id?: string;
|
||||
voiceId?: string;
|
||||
model?: string;
|
||||
model_id?: string;
|
||||
voice_settings?: unknown;
|
||||
voiceSettings?: unknown;
|
||||
seed?: number;
|
||||
remove_background_noise?: boolean;
|
||||
output_format?: string;
|
||||
file_format?: string;
|
||||
optimize_streaming_latency?: number;
|
||||
enable_logging?: boolean;
|
||||
test_mode?: boolean;
|
||||
}
|
||||
|
||||
export class VoiceChangerDriver extends PuterDriver {
|
||||
readonly driverInterface = 'puter-speech2speech';
|
||||
readonly driverName = 'ai-speech2speech';
|
||||
// Older SDK bundles name the provider in the driver slot.
|
||||
readonly driverAliases = ['elevenlabs-voice-changer'];
|
||||
readonly isDefault = true;
|
||||
|
||||
// Shared AI policy — see `drivers/util/aiLimits.ts` for the tier table.
|
||||
readonly rateLimit = AI_RATE_LIMIT;
|
||||
readonly concurrent = AI_CONCURRENT;
|
||||
|
||||
/** Metering scoped to this driver. Lazy: services wire up after drivers. */
|
||||
get #aiMetering(): AiMeteringService {
|
||||
return withAiCostFactor(
|
||||
this.services.metering,
|
||||
this.clients.event,
|
||||
this.driverName,
|
||||
);
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(VOICE_CHANGER_COSTS).map(
|
||||
([usageType, ucentsPerUnit]) => ({
|
||||
usageType,
|
||||
ucentsPerUnit,
|
||||
unit: 'second',
|
||||
source: 'driver:aiSpeech2Speech',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
#providers = new ProviderRegistry<ElevenLabsConfig>(VOICE_CHANGER_CATALOG);
|
||||
|
||||
override onServerStart() {
|
||||
const elevenlabs = this.config.providers?.elevenlabs;
|
||||
const apiKey = readProviderKey(elevenlabs);
|
||||
if (!apiKey) return;
|
||||
this.#providers.register('elevenlabs', {
|
||||
apiKey,
|
||||
baseUrl:
|
||||
(elevenlabs?.apiBaseUrl as string | undefined) ??
|
||||
'https://api.elevenlabs.io',
|
||||
defaultVoiceId:
|
||||
(elevenlabs?.defaultVoiceId as string | undefined) ??
|
||||
DEFAULT_VOICE_ID,
|
||||
defaultModelId:
|
||||
(elevenlabs?.speechToSpeechModelId as string | undefined) ??
|
||||
DEFAULT_MODEL,
|
||||
});
|
||||
}
|
||||
|
||||
async convert(
|
||||
args: ConvertArgs,
|
||||
): Promise<DriverStreamResult | { url: string; content_type: string }> {
|
||||
// Only one provider exists today, but naming a different one should
|
||||
// fail loudly rather than quietly convert with this one.
|
||||
const providerName = this.#providers.resolve(args.provider);
|
||||
|
||||
if (args.test_mode) {
|
||||
return { url: SAMPLE_AUDIO_URL, content_type: 'audio/mpeg' };
|
||||
}
|
||||
|
||||
const elevenlabs = this.#providers.get(providerName);
|
||||
if (!elevenlabs) {
|
||||
throw new HttpError(500, 'ElevenLabs API key not configured', {
|
||||
legacyCode: 'internal_error',
|
||||
});
|
||||
}
|
||||
|
||||
const actor = Context.get('actor');
|
||||
if (!actor)
|
||||
throw new HttpError(401, 'Authentication required', {
|
||||
legacyCode: 'unauthorized',
|
||||
});
|
||||
|
||||
if (!args.audio) {
|
||||
throw new HttpError(400, '`audio` is required', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
|
||||
const modelId =
|
||||
args.model_id || args.model || elevenlabs.defaultModelId;
|
||||
const voiceId =
|
||||
args.voice_id ||
|
||||
args.voiceId ||
|
||||
args.voice ||
|
||||
elevenlabs.defaultVoiceId;
|
||||
if (!voiceId)
|
||||
throw new HttpError(400, '`voice` is required', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
// `voiceId` lands in the request URL path, which carries our API key,
|
||||
// so anything outside ElevenLabs' `[A-Za-z0-9_-]+` id shape could
|
||||
// steer the request at a different endpoint.
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(voiceId))
|
||||
throw new HttpError(400, '`voice` must be alphanumeric', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
|
||||
// Gate on the cost table: an id we can't price is one we can't bill
|
||||
// for, and ElevenLabs bills us for it either way. This also keeps
|
||||
// anything but a known id out of the multipart `model_id` field.
|
||||
const usageKey = `elevenlabs:${modelId}:second`;
|
||||
if (!Object.hasOwn(VOICE_CHANGER_COSTS, usageKey)) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`Invalid model: ${modelId}. Expected: ${SUPPORTED_MODELS.join(', ')}`,
|
||||
{
|
||||
legacyCode: 'field_invalid',
|
||||
fields: {
|
||||
key: 'model',
|
||||
expected: SUPPORTED_MODELS,
|
||||
got: modelId,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
const ucentsPerSecond = VOICE_CHANGER_COSTS[usageKey]!;
|
||||
|
||||
const loaded = await loadFileInput(
|
||||
this.stores,
|
||||
this.services.fs,
|
||||
actor,
|
||||
args.audio,
|
||||
{ maxBytes: MAX_AUDIO_FILE_SIZE, acceptWebInput: true },
|
||||
);
|
||||
|
||||
// The hold estimates seconds from the input at 128 kbps: compact audio
|
||||
// is under-held rather than refused. Usage is metered from the length
|
||||
// of the audio that comes back.
|
||||
const estimatedSeconds = Math.max(
|
||||
1,
|
||||
Math.ceil(loaded.buffer.byteLength / 16000),
|
||||
);
|
||||
const estimatedCost = ucentsPerSecond * estimatedSeconds;
|
||||
|
||||
const formData = new FormData();
|
||||
const blob = new Blob([loaded.buffer as BlobPart], {
|
||||
type: loaded.mimeType ?? 'application/octet-stream',
|
||||
});
|
||||
formData.append('audio', blob, loaded.filename);
|
||||
formData.append('model_id', modelId);
|
||||
|
||||
const settings = args.voice_settings ?? args.voiceSettings;
|
||||
if (settings !== undefined && settings !== null) {
|
||||
formData.append(
|
||||
'voice_settings',
|
||||
typeof settings === 'string'
|
||||
? settings
|
||||
: JSON.stringify(settings),
|
||||
);
|
||||
}
|
||||
if (args.seed !== undefined && args.seed !== null) {
|
||||
formData.append('seed', String(args.seed));
|
||||
}
|
||||
if (typeof args.remove_background_noise === 'boolean') {
|
||||
formData.append(
|
||||
'remove_background_noise',
|
||||
String(args.remove_background_noise),
|
||||
);
|
||||
}
|
||||
if (args.file_format) {
|
||||
formData.append('file_format', args.file_format);
|
||||
}
|
||||
|
||||
const searchParams = new URLSearchParams();
|
||||
const outputFormat = args.output_format || DEFAULT_OUTPUT_FORMAT;
|
||||
if (outputFormat) searchParams.set('output_format', outputFormat);
|
||||
if (
|
||||
args.optimize_streaming_latency !== undefined &&
|
||||
args.optimize_streaming_latency !== null
|
||||
) {
|
||||
searchParams.set(
|
||||
'optimize_streaming_latency',
|
||||
String(args.optimize_streaming_latency),
|
||||
);
|
||||
}
|
||||
if (args.enable_logging !== undefined && args.enable_logging !== null) {
|
||||
searchParams.set('enable_logging', String(args.enable_logging));
|
||||
}
|
||||
|
||||
const url = new URL(
|
||||
`/v1/speech-to-speech/${voiceId}`,
|
||||
elevenlabs.baseUrl,
|
||||
);
|
||||
const search = searchParams.toString();
|
||||
if (search) url.search = search;
|
||||
|
||||
const metering = this.#aiMetering;
|
||||
const output = measuredOutput(
|
||||
outputBytesPerSecond(outputFormat),
|
||||
outputFormat,
|
||||
);
|
||||
return withStreamedAiCharge(
|
||||
metering,
|
||||
actor,
|
||||
usageKey,
|
||||
estimatedCost,
|
||||
async (): Promise<DriverStreamResult> => {
|
||||
const response = await upstreamFetch(
|
||||
'elevenlabs',
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'xi-api-key': elevenlabs.apiKey },
|
||||
body: formData,
|
||||
},
|
||||
{
|
||||
providerLabel: 'ElevenLabs',
|
||||
timeoutMs: CONVERT_TIMEOUT_MS,
|
||||
stream: true,
|
||||
},
|
||||
);
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type:
|
||||
response.headers.get('content-type') ?? 'audio/mpeg',
|
||||
stream: output.measure(upstreamBodyStream(response)),
|
||||
};
|
||||
},
|
||||
async () => {
|
||||
const measured = await output.seconds;
|
||||
// A stream cut short was still converted in full upstream.
|
||||
const seconds =
|
||||
measured.complete && measured.seconds
|
||||
? measured.seconds
|
||||
: Math.max(measured.seconds ?? 0, estimatedSeconds);
|
||||
return metering.incrementUsage(
|
||||
actor,
|
||||
usageKey,
|
||||
seconds,
|
||||
ucentsPerSecond * seconds,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes per second of an ElevenLabs `output_format`. A format outside the
|
||||
* documented set is sized as the default, so a made-up bitrate can't shrink the
|
||||
* bill.
|
||||
*/
|
||||
export function outputBytesPerSecond(format: string): number {
|
||||
const known = OUTPUT_FORMATS.has(format) ? format : DEFAULT_OUTPUT_FORMAT;
|
||||
const [codec, rate, kbps] = known.split('_');
|
||||
if (codec === 'mp3' || codec === 'opus') return Number(kbps) * 125;
|
||||
if (codec === 'ulaw' || codec === 'alaw') return Number(rate);
|
||||
// pcm and wav: 16-bit mono.
|
||||
return Number(rate) * 2;
|
||||
}
|
||||
|
||||
// Past this, a WAV whose data chunk hasn't started is billed by raw length.
|
||||
const WAV_HEADER_MAX_BYTES = 64 * 1024;
|
||||
|
||||
/**
|
||||
* Where the samples of a RIFF/WAVE stream start, once `head` reaches its `data`
|
||||
* chunk; undefined until then, or if `head` isn't a WAV.
|
||||
*/
|
||||
export function wavDataOffset(head: Buffer): number | undefined {
|
||||
if (head.length < 12) return undefined;
|
||||
if (
|
||||
head.toString('ascii', 0, 4) !== 'RIFF' ||
|
||||
head.toString('ascii', 8, 12) !== 'WAVE'
|
||||
)
|
||||
return undefined;
|
||||
let offset = 12;
|
||||
while (offset + 8 <= head.length) {
|
||||
if (head.toString('ascii', offset, offset + 4) === 'data')
|
||||
return offset + 8;
|
||||
const size = head.readUInt32LE(offset + 4);
|
||||
offset += 8 + size + (size & 1);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const OGG_HEADER_BYTES = 27;
|
||||
const NO_GRANULE = 2n ** 64n - 1n;
|
||||
|
||||
/**
|
||||
* Walks an Ogg Opus stream page by page as it arrives. Opus is variable
|
||||
* bitrate, so its length comes from the last page's granule position (48 kHz
|
||||
* samples, less the stream's pre-skip) rather than its byte count.
|
||||
*/
|
||||
export class OggOpusClock {
|
||||
#header = Buffer.alloc(0);
|
||||
#skip = 0;
|
||||
#firstBody: Buffer | undefined = Buffer.alloc(0);
|
||||
#firstBodyBytes = 0;
|
||||
#granule: bigint | undefined;
|
||||
#preSkip = 0;
|
||||
#broken = false;
|
||||
|
||||
push(chunk: Buffer): void {
|
||||
let offset = 0;
|
||||
while (!this.#broken && offset < chunk.length) {
|
||||
if (this.#skip > 0) {
|
||||
const n = Math.min(this.#skip, chunk.length - offset);
|
||||
if (this.#firstBody && this.#firstBody.length < 19) {
|
||||
this.#firstBody = Buffer.concat([
|
||||
this.#firstBody,
|
||||
chunk.subarray(offset, offset + n),
|
||||
]);
|
||||
}
|
||||
this.#skip -= n;
|
||||
offset += n;
|
||||
if (this.#skip === 0 && this.#firstBody) this.#readOpusHead();
|
||||
continue;
|
||||
}
|
||||
const want =
|
||||
this.#header.length < OGG_HEADER_BYTES
|
||||
? OGG_HEADER_BYTES
|
||||
: OGG_HEADER_BYTES + this.#header[26]!;
|
||||
const n = Math.min(
|
||||
want - this.#header.length,
|
||||
chunk.length - offset,
|
||||
);
|
||||
this.#header = Buffer.concat([
|
||||
this.#header,
|
||||
chunk.subarray(offset, offset + n),
|
||||
]);
|
||||
offset += n;
|
||||
if (this.#header.length < want) continue;
|
||||
if (this.#header.toString('ascii', 0, 4) !== 'OggS') {
|
||||
this.#broken = true;
|
||||
return;
|
||||
}
|
||||
if (want === OGG_HEADER_BYTES && this.#header[26]! > 0) continue;
|
||||
const granule = this.#header.readBigUInt64LE(6);
|
||||
if (granule !== NO_GRANULE) this.#granule = granule;
|
||||
let body = 0;
|
||||
for (const lacing of this.#header.subarray(OGG_HEADER_BYTES)) {
|
||||
body += lacing;
|
||||
}
|
||||
this.#header = Buffer.alloc(0);
|
||||
this.#skip = body;
|
||||
if (this.#firstBody) this.#firstBodyBytes = body;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whole seconds so far, once the stream has proven to be Ogg Opus. */
|
||||
get seconds(): number | undefined {
|
||||
if (this.#broken || this.#granule === undefined) return undefined;
|
||||
const samples = Number(this.#granule) - this.#preSkip;
|
||||
return samples > 0
|
||||
? Math.max(1, Math.ceil(samples / 48_000))
|
||||
: undefined;
|
||||
}
|
||||
|
||||
#readOpusHead(): void {
|
||||
const head = this.#firstBody!;
|
||||
this.#firstBody = undefined;
|
||||
if (
|
||||
this.#firstBodyBytes < 19 ||
|
||||
head.toString('ascii', 0, 8) !== 'OpusHead'
|
||||
) {
|
||||
this.#broken = true;
|
||||
return;
|
||||
}
|
||||
this.#preSkip = head.readUInt16LE(10);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the bytes of a converted stream. `seconds` settles when the stream
|
||||
* ends or closes: its length in whole seconds (undefined when nothing came
|
||||
* back), and whether the upstream sent all of it. A WAV's header isn't audio,
|
||||
* so it isn't counted; Opus is timed by its pages.
|
||||
*/
|
||||
function measuredOutput(bytesPerSecond: number, format: string) {
|
||||
let bytes = 0;
|
||||
let head: Buffer | undefined = format.startsWith('wav_')
|
||||
? Buffer.alloc(0)
|
||||
: undefined;
|
||||
const opus = format.startsWith('opus_') ? new OggOpusClock() : undefined;
|
||||
let headerBytes = 0;
|
||||
let settle!: (measured: { seconds?: number; complete: boolean }) => void;
|
||||
const seconds = new Promise<{ seconds?: number; complete: boolean }>(
|
||||
(resolve) => {
|
||||
settle = resolve;
|
||||
},
|
||||
);
|
||||
const lengthOf = () => {
|
||||
const timed = opus?.seconds;
|
||||
if (timed !== undefined) return timed;
|
||||
const audio = bytes - headerBytes;
|
||||
return audio > 0
|
||||
? Math.max(1, Math.ceil(audio / bytesPerSecond))
|
||||
: undefined;
|
||||
};
|
||||
const measure = (source: Readable): Readable => {
|
||||
const counter = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
bytes += chunk.length;
|
||||
opus?.push(chunk);
|
||||
if (head) {
|
||||
head = Buffer.concat([head, chunk]);
|
||||
const offset = wavDataOffset(head);
|
||||
if (offset !== undefined) headerBytes = offset;
|
||||
if (
|
||||
offset !== undefined ||
|
||||
head.length >= WAV_HEADER_MAX_BYTES
|
||||
)
|
||||
head = undefined;
|
||||
}
|
||||
callback(null, chunk);
|
||||
},
|
||||
flush(callback) {
|
||||
settle({ seconds: lengthOf(), complete: true });
|
||||
callback();
|
||||
},
|
||||
});
|
||||
counter.once('close', () =>
|
||||
settle({ seconds: lengthOf(), complete: false }),
|
||||
);
|
||||
pipeline(source, counter, () => {});
|
||||
return counter;
|
||||
};
|
||||
return { measure, seconds };
|
||||
}
|
||||
+13
-2
@@ -19,9 +19,9 @@
|
||||
|
||||
import { createServer, type Server } from 'node:http';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../../../testUtil.js';
|
||||
import { clearUnhealthyRoutes } from './utils/providerHealth.js';
|
||||
import { makeActor } from '../../core/actor.js';
|
||||
import { makeActor } from '../../../../core/actor.js';
|
||||
|
||||
const policyMessage =
|
||||
'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://vendor.test/private';
|
||||
@@ -129,6 +129,17 @@ describe('model availability through real HTTP', () => {
|
||||
meteringEnforcement: { subscriptions: false },
|
||||
} as never);
|
||||
}, 120000);
|
||||
it('serves qualified chat model names through HTTP', async () => {
|
||||
const response = await fetch(
|
||||
new URL('/puterai/chat/models', env.apiOrigin),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
const { models } = (await response.json()) as { models: string[] };
|
||||
expect(models).toContain('openrouter:vendor/availability-test');
|
||||
expect(models.every((id) => /^[^:/]+:[^/]+\/.+/.test(id))).toBe(true);
|
||||
expect(models).not.toContain('vendor/availability-test');
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
upstream?.closeAllConnections();
|
||||
+2
-2
@@ -19,8 +19,8 @@
|
||||
|
||||
import { createServer, type Server, type ServerResponse } from 'node:http';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { makeActor, type Actor } from '../../core/actor.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
import { makeActor, type Actor } from '../../../../core/actor.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../../../testUtil.js';
|
||||
|
||||
const routes = [
|
||||
'/drivers/call',
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { createServer, type Server, type ServerResponse } from 'node:http';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../../../testUtil.js';
|
||||
|
||||
/**
|
||||
* A completion runs on to its usage report after the caller hangs up, so it
|
||||
* keeps counting against the caller's concurrency cap until it settles.
|
||||
*/
|
||||
describe('chat concurrency over HTTP after the caller hangs up', () => {
|
||||
let upstream: Server;
|
||||
let env: PuterTestEnv;
|
||||
let onRequest: (res: ServerResponse, body: string) => void;
|
||||
|
||||
beforeAll(async () => {
|
||||
upstream = createServer((req, res) => {
|
||||
if (req.url === '/api/tags') {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ models: [{ name: 'slot-test' }] }));
|
||||
return;
|
||||
}
|
||||
let body = '';
|
||||
req.setEncoding('utf8');
|
||||
req.on('data', (chunk: string) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.once('end', () => onRequest(res, body));
|
||||
});
|
||||
await new Promise<void>((resolve) =>
|
||||
upstream.listen(0, '127.0.0.1', resolve),
|
||||
);
|
||||
const port = (upstream.address() as { port: number }).port;
|
||||
env = await setupPuterTestEnv({
|
||||
providers: {
|
||||
ollama: { apiBaseUrl: `http://127.0.0.1:${port}` },
|
||||
},
|
||||
meteringEnforcement: { subscriptions: false },
|
||||
} as never);
|
||||
await vi.waitFor(async () => {
|
||||
expect(
|
||||
(await env.server.drivers.aiChat.models()).some(
|
||||
(model) => model.id === 'ollama:ollama/slot-test',
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
upstream?.closeAllConnections();
|
||||
if (upstream)
|
||||
await new Promise<void>((resolve) =>
|
||||
upstream.close(() => resolve()),
|
||||
);
|
||||
});
|
||||
|
||||
const call = (route: string, stream: boolean, signal?: AbortSignal) => {
|
||||
const args = {
|
||||
model: 'ollama:ollama/slot-test',
|
||||
stream,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
max_tokens: 16,
|
||||
};
|
||||
return fetch(new URL(route, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${env.users.user.apiToken}`,
|
||||
Origin: env.apiOrigin,
|
||||
},
|
||||
body: JSON.stringify(
|
||||
route === '/drivers/call'
|
||||
? {
|
||||
interface: 'puter-chat-completion',
|
||||
method: 'complete',
|
||||
args,
|
||||
}
|
||||
: args,
|
||||
),
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
const usage = { prompt_tokens: 3, completion_tokens: 5, total_tokens: 8 };
|
||||
const chunk = (delta: object, finish: string | null, extra = {}) =>
|
||||
`data: ${JSON.stringify({
|
||||
id: 'chatcmpl_1',
|
||||
object: 'chat.completion.chunk',
|
||||
choices: [{ index: 0, delta, finish_reason: finish }],
|
||||
...extra,
|
||||
})}\n\n`;
|
||||
|
||||
const finishAnswer = (res: ServerResponse, body: string) => {
|
||||
if (!(JSON.parse(body) as { stream?: boolean }).stream) {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
id: 'chatcmpl_1',
|
||||
object: 'chat.completion',
|
||||
created: 0,
|
||||
model: 'slot-test',
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: 'done' },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!res.headersSent)
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.end(
|
||||
`${chunk({ content: 'done' }, 'stop')}data: ${JSON.stringify({
|
||||
id: 'chatcmpl_1',
|
||||
object: 'chat.completion.chunk',
|
||||
choices: [],
|
||||
usage,
|
||||
})}\n\ndata: [DONE]\n\n`,
|
||||
);
|
||||
};
|
||||
|
||||
/** Starts a completion, waits until it is generating, then hangs up. */
|
||||
const abandon = async (route: string, stream: boolean) => {
|
||||
let received!: () => void;
|
||||
const reached = new Promise<void>((resolve) => {
|
||||
received = resolve;
|
||||
});
|
||||
const pending: { res: ServerResponse; body: string }[] = [];
|
||||
onRequest = (res, body) => {
|
||||
pending.push({ res, body });
|
||||
if (stream) {
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.write(chunk({ role: 'assistant', content: 'partial' }, null));
|
||||
}
|
||||
received();
|
||||
};
|
||||
const abort = new AbortController();
|
||||
const request = call(route, stream, abort.signal);
|
||||
const outcome = await Promise.race([
|
||||
reached.then(() => 'generating' as const),
|
||||
request.then((response) => response),
|
||||
]);
|
||||
if (outcome !== 'generating') {
|
||||
return { refused: outcome, pending };
|
||||
}
|
||||
if (stream) {
|
||||
const response = await request;
|
||||
const reader = response.body!.getReader();
|
||||
let text = '';
|
||||
while (!text.includes('partial')) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
text += new TextDecoder().decode(value);
|
||||
}
|
||||
}
|
||||
abort.abort();
|
||||
await request.catch(() => undefined);
|
||||
return { refused: undefined, pending };
|
||||
};
|
||||
|
||||
it.each([
|
||||
{ route: '/drivers/call', stream: true },
|
||||
{ route: '/drivers/call', stream: false },
|
||||
{ route: '/puterai/openai/v1/chat/completions', stream: true },
|
||||
{ route: '/puterai/openai/v1/chat/completions', stream: false },
|
||||
{ route: '/puterai/anthropic/v1/messages', stream: true },
|
||||
])(
|
||||
'counts an abandoned completion until it settles: $route (stream=$stream)',
|
||||
async ({ route, stream }) => {
|
||||
const running: { res: ServerResponse; body: string }[] = [];
|
||||
let refused: Response | undefined;
|
||||
try {
|
||||
for (let i = 0; i < 25 && !refused; i++) {
|
||||
const attempt = await abandon(route, stream);
|
||||
running.push(...attempt.pending);
|
||||
refused = attempt.refused;
|
||||
// Let the server see the hang-up before the next call.
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
expect(refused?.status).toBe(429);
|
||||
expect(await refused!.text()).toContain(
|
||||
'Too many concurrent requests',
|
||||
);
|
||||
expect(running.length).toBeGreaterThan(0);
|
||||
|
||||
for (const { res, body } of running.splice(0))
|
||||
finishAnswer(res, body);
|
||||
onRequest = finishAnswer;
|
||||
await expect
|
||||
.poll(async () => (await call(route, false)).status, {
|
||||
timeout: 15_000,
|
||||
})
|
||||
.toBe(200);
|
||||
} finally {
|
||||
for (const { res, body } of running) finishAnswer(res, body);
|
||||
upstream.closeAllConnections();
|
||||
// The next case starts with a fresh rate-limit window.
|
||||
await new Promise((resolve) => setTimeout(resolve, 10_100));
|
||||
}
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
+52
-15
@@ -42,22 +42,22 @@ import {
|
||||
vi,
|
||||
} from 'vitest';
|
||||
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { withTestActor } from '../integrationTestUtil.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../shared/integrationTestUtil.js';
|
||||
import { ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import { AzureChatProvider } from './providers/azure/AzureChatProvider.js';
|
||||
import { FakeChatProvider } from './providers/FakeChatProvider.js';
|
||||
import { InfronProvider } from './providers/infron/InfronProvider.js';
|
||||
import { NeuralwattProvider } from './providers/neuralwatt/NeuralwattProvider.js';
|
||||
import { OpenAiChatProvider } from './providers/openai/OpenAiChatCompletionsProvider.js';
|
||||
import { OpenRouterProvider } from './providers/openrouter/OpenRouterProvider.js';
|
||||
import { TogetherAIProvider } from './providers/together/TogetherAIProvider.js';
|
||||
import { XAI_CHAT_MODELS } from './providers/xai/catalog.js';
|
||||
import { XAIProvider } from './providers/xai/XAIProvider.js';
|
||||
import { handle_completion_output } from './utils/OpenAIUtil.js';
|
||||
import { AzureChatProvider } from '../../providers/azure/chat/AzureChatProvider.js';
|
||||
import { FakeChatProvider } from '../../providers/fake/chat/FakeChatProvider.js';
|
||||
import { InfronProvider } from '../../providers/infron/chat/InfronProvider.js';
|
||||
import { NeuralwattProvider } from '../../providers/neuralwatt/chat/NeuralwattProvider.js';
|
||||
import { OpenAiChatProvider } from '../../providers/openai/chat/OpenAiChatCompletionsProvider.js';
|
||||
import { OpenRouterProvider } from '../../providers/openrouter/chat/OpenRouterProvider.js';
|
||||
import { TogetherAIProvider } from '../../providers/together/chat/TogetherAIProvider.js';
|
||||
import { XAI_CHAT_MODELS } from '../../providers/xai/chat/catalog.js';
|
||||
import { XAIProvider } from '../../providers/xai/chat/XAIProvider.js';
|
||||
import { handle_completion_output } from '../../protocols/openaiCompatible/OpenAIUtil.js';
|
||||
import { clearUnhealthyRoutes, isRouteUnhealthy } from './utils/providerHealth.js';
|
||||
|
||||
let server: PuterServer;
|
||||
@@ -1231,4 +1231,41 @@ describe('ChatCompletionDriver streaming failure handling', () => {
|
||||
expect(events).toContainEqual({ type: 'text', text: 'hello' });
|
||||
expect(cleanup).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('frees the concurrency slot before the provider cleanup hook finishes', async () => {
|
||||
let finishCleanup!: () => void;
|
||||
const cleanup = vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishCleanup = resolve;
|
||||
}),
|
||||
);
|
||||
vi.spyOn(FakeChatProvider.prototype, 'complete').mockResolvedValue({
|
||||
stream: true,
|
||||
init_chat_stream: async ({
|
||||
chatStream,
|
||||
}: {
|
||||
chatStream: { end: (usage?: Record<string, number>) => void };
|
||||
}) => {
|
||||
chatStream.end({ input_tokens: 1, output_tokens: 1 });
|
||||
},
|
||||
finally_fn: cleanup,
|
||||
} as never);
|
||||
|
||||
const kept: Promise<unknown>[] = [];
|
||||
const result = (await withTestActor(() => {
|
||||
Context.set('keepConcurrencySlot', (work) => kept.push(work));
|
||||
return fakeOnlyDriver.complete({
|
||||
model: 'fake',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
stream: true,
|
||||
} as never);
|
||||
})) as unknown as { stream: Readable };
|
||||
await collect(result.stream);
|
||||
|
||||
expect(kept).toHaveLength(1);
|
||||
await kept[0];
|
||||
await vi.waitFor(() => expect(cleanup).toHaveBeenCalledTimes(1));
|
||||
finishCleanup();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,726 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Puter is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with Puter. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Seeded fuzzing of `ChatCompletionDriver.complete` through every registered
|
||||
* provider. The vendors are played at the HTTP boundary, so the real SDKs and
|
||||
* the providers' own request and response code run.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { isHttpError } from '../../../../core/http/HttpError.js';
|
||||
import type { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { kv } from '../../../../util/kvSingleton.js';
|
||||
import { isDriverStreamResult } from '../../../meta.js';
|
||||
import {
|
||||
caseRandom,
|
||||
corruptReply,
|
||||
Findings,
|
||||
fuzzCases,
|
||||
hostileValue,
|
||||
mutate,
|
||||
pick,
|
||||
pickStable,
|
||||
type Random,
|
||||
unclassifiedFailure,
|
||||
upstream,
|
||||
type UpstreamReply,
|
||||
type UpstreamRequest,
|
||||
} from '../../shared/fuzzTestUtil.js';
|
||||
import {
|
||||
readStreamResult,
|
||||
withTestActor,
|
||||
} from '../../shared/integrationTestUtil.js';
|
||||
import { ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import type { ICompleteArguments } from './types.js';
|
||||
import { clearUnhealthyRoutes } from './utils/providerHealth.js';
|
||||
|
||||
vi.mock('undici', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('undici')>()),
|
||||
fetch: (input: unknown, init?: RequestInit) => upstream.fetch(input, init),
|
||||
}));
|
||||
|
||||
// groq-sdk brings its own HTTP client. Groq's API is OpenAI-compatible, so the
|
||||
// OpenAI SDK stands in for it and the requests reach the stub like the rest.
|
||||
vi.mock('groq-sdk', async () => ({ default: (await import('openai')).OpenAI }));
|
||||
|
||||
vi.mock('axios', () => {
|
||||
const request = (config: Parameters<typeof upstream.axios>[0]) =>
|
||||
upstream.axios(config);
|
||||
const get = (url: string, config?: object) =>
|
||||
upstream.axios({ ...config, url, method: 'GET' });
|
||||
return { default: { request, get }, request, get };
|
||||
});
|
||||
|
||||
// -- The vendors, played at the HTTP boundary --
|
||||
|
||||
const TEXT = 'pong';
|
||||
const chatUsage = { prompt_tokens: 3, completion_tokens: 5, total_tokens: 8 };
|
||||
const chatChunk = (delta: object, finish: string | null) => ({
|
||||
id: 'chatcmpl-1',
|
||||
object: 'chat.completion.chunk',
|
||||
created: 1,
|
||||
model: 'm',
|
||||
choices: [{ index: 0, delta, finish_reason: finish }],
|
||||
});
|
||||
const responsesMessage = {
|
||||
type: 'message',
|
||||
id: 'msg_1',
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
content: [{ type: 'output_text', text: TEXT, annotations: [] }],
|
||||
};
|
||||
const responsesBody = {
|
||||
id: 'resp_1',
|
||||
object: 'response',
|
||||
status: 'completed',
|
||||
model: 'm',
|
||||
output: [responsesMessage],
|
||||
usage: {
|
||||
input_tokens: 3,
|
||||
output_tokens: 5,
|
||||
total_tokens: 8,
|
||||
input_tokens_details: { cached_tokens: 0 },
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
};
|
||||
const anthropicUsage = { input_tokens: 3, output_tokens: 5 };
|
||||
const geminiBody = {
|
||||
candidates: [
|
||||
{
|
||||
index: 0,
|
||||
content: { role: 'model', parts: [{ text: TEXT }] },
|
||||
finishReason: 'STOP',
|
||||
},
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 3,
|
||||
candidatesTokenCount: 5,
|
||||
totalTokenCount: 8,
|
||||
},
|
||||
};
|
||||
|
||||
const anthropicEvents = [
|
||||
{
|
||||
type: 'message_start',
|
||||
message: {
|
||||
id: 'msg_1',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'm',
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: { input_tokens: 3, output_tokens: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'content_block_start',
|
||||
index: 0,
|
||||
content_block: { type: 'text', text: '' },
|
||||
},
|
||||
{
|
||||
type: 'content_block_delta',
|
||||
index: 0,
|
||||
delta: { type: 'text_delta', text: TEXT },
|
||||
},
|
||||
{ type: 'content_block_stop', index: 0 },
|
||||
{
|
||||
type: 'message_delta',
|
||||
delta: { stop_reason: 'end_turn', stop_sequence: null },
|
||||
usage: { output_tokens: 5 },
|
||||
},
|
||||
{ type: 'message_stop' },
|
||||
];
|
||||
|
||||
const responsesEvents = [
|
||||
{
|
||||
type: 'response.created',
|
||||
response: { ...responsesBody, status: 'in_progress', output: [] },
|
||||
},
|
||||
{
|
||||
type: 'response.output_item.added',
|
||||
output_index: 0,
|
||||
item: { ...responsesMessage, status: 'in_progress', content: [] },
|
||||
},
|
||||
{
|
||||
type: 'response.output_text.delta',
|
||||
item_id: 'msg_1',
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
delta: TEXT,
|
||||
},
|
||||
{
|
||||
type: 'response.output_item.done',
|
||||
output_index: 0,
|
||||
item: responsesMessage,
|
||||
},
|
||||
{ type: 'response.completed', response: responsesBody },
|
||||
];
|
||||
|
||||
/** A well-formed answer from whichever vendor API the request went to. */
|
||||
const healthyUpstream = ({
|
||||
url,
|
||||
json,
|
||||
}: UpstreamRequest): UpstreamReply | undefined => {
|
||||
const path = url.pathname;
|
||||
// A caller-supplied media link that leads nowhere.
|
||||
if (url.hostname === 'upstream.test') return { status: 404, body: '' };
|
||||
const streaming = (json as { stream?: unknown } | undefined)?.stream;
|
||||
if (path.endsWith('/chat/completions')) {
|
||||
if (!streaming) {
|
||||
return {
|
||||
body: {
|
||||
id: 'chatcmpl-1',
|
||||
object: 'chat.completion',
|
||||
created: 1,
|
||||
model: 'm',
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: TEXT },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: chatUsage,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
sse: [
|
||||
{ data: chatChunk({ role: 'assistant', content: TEXT }, null) },
|
||||
{ data: chatChunk({}, 'stop') },
|
||||
{
|
||||
data: {
|
||||
...chatChunk({}, null),
|
||||
choices: [],
|
||||
usage: chatUsage,
|
||||
},
|
||||
},
|
||||
{ data: '[DONE]' },
|
||||
],
|
||||
};
|
||||
}
|
||||
if (path.endsWith('/responses')) {
|
||||
return streaming
|
||||
? {
|
||||
sse: responsesEvents.map((data) => ({
|
||||
event: data.type,
|
||||
data,
|
||||
})),
|
||||
}
|
||||
: { body: responsesBody };
|
||||
}
|
||||
if (path.endsWith('/messages/count_tokens')) {
|
||||
return { body: { input_tokens: 3 } };
|
||||
}
|
||||
if (path.endsWith('/messages')) {
|
||||
return streaming
|
||||
? {
|
||||
sse: anthropicEvents.map((data) => ({
|
||||
event: data.type,
|
||||
data,
|
||||
})),
|
||||
}
|
||||
: {
|
||||
body: {
|
||||
id: 'msg_1',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'm',
|
||||
content: [{ type: 'text', text: TEXT }],
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
usage: anthropicUsage,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (path.endsWith(':generateContent')) return { body: geminiBody };
|
||||
if (path.endsWith(':streamGenerateContent')) {
|
||||
return { sse: [{ data: geminiBody }] };
|
||||
}
|
||||
if (path.endsWith(':countTokens')) return { body: { totalTokens: 3 } };
|
||||
if (path.endsWith('/moderations')) {
|
||||
return {
|
||||
body: {
|
||||
id: 'modr_1',
|
||||
model: 'm',
|
||||
results: [
|
||||
{ flagged: false, categories: {}, category_scores: {} },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
// -- Requests to mutate --
|
||||
|
||||
const PNG =
|
||||
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
|
||||
|
||||
const weatherTool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_weather',
|
||||
description: 'Get the weather for a city',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { city: { type: 'string' } },
|
||||
required: ['city'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const CORPUS: Record<string, unknown>[] = [
|
||||
{ messages: [{ role: 'user', content: 'hi' }] },
|
||||
{
|
||||
messages: [
|
||||
{ role: 'system', content: 'Be brief.' },
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'What is this?' },
|
||||
{ type: 'image_url', image_url: { url: PNG } },
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens: 64,
|
||||
temperature: 0.5,
|
||||
top_p: 0.9,
|
||||
stream: true,
|
||||
},
|
||||
{
|
||||
messages: [
|
||||
{ role: 'user', content: 'Weather in Paris?' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'call_1',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_weather',
|
||||
arguments: '{"city":"Paris"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{ role: 'tool', tool_call_id: 'call_1', content: 'Sunny' },
|
||||
],
|
||||
tools: [weatherTool],
|
||||
tool_choice: 'auto',
|
||||
parallel_tool_calls: false,
|
||||
},
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Weather in Paris?',
|
||||
cache_control: { type: 'ephemeral' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'text', text: 'Checking.' },
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'toolu_1',
|
||||
name: 'get_weather',
|
||||
input: { city: 'Paris' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'toolu_1',
|
||||
content: 'Sunny',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
tools: [weatherTool, { type: 'web_search' }],
|
||||
tool_choice: { type: 'function', function: { name: 'get_weather' } },
|
||||
stream: true,
|
||||
streamToolInput: true,
|
||||
},
|
||||
{
|
||||
messages: [{ role: 'user', content: 'Reply in JSON.' }],
|
||||
outputFormat: {
|
||||
type: 'json_schema',
|
||||
schema: { type: 'object', properties: { a: { type: 'string' } } },
|
||||
},
|
||||
reasoning: { effort: 'low' },
|
||||
reasoning_effort: 'low',
|
||||
thinking: { type: 'enabled', budget_tokens: 1024 },
|
||||
text: { verbosity: 'low' },
|
||||
verbosity: 'low',
|
||||
stopSequences: ['END'],
|
||||
topK: 5,
|
||||
normalize: true,
|
||||
},
|
||||
{
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
instructions: 'Be brief.',
|
||||
compaction: { trigger_tokens: 150000 },
|
||||
context_management: { edits: [{ type: 'clear_tool_uses_20250919' }] },
|
||||
cacheControl: { type: 'ephemeral', ttl: '1h' },
|
||||
prompt_cache_key: 'k',
|
||||
prompt_cache_retention: '24h',
|
||||
serviceSpeed: 'fast',
|
||||
service_tier: 'flex',
|
||||
speed: 'fast',
|
||||
metadata: { trace: 't' },
|
||||
include: ['reasoning.encrypted_content'],
|
||||
store: false,
|
||||
truncation: 'auto',
|
||||
taskBudget: { total: 1000, remaining: 500 },
|
||||
safeguards: [{ type: 'prompt_injection' }],
|
||||
anthropicBetas: ['context-management-2025-06-27'],
|
||||
moderation: true,
|
||||
response: { normalize: true },
|
||||
fallback: false,
|
||||
},
|
||||
];
|
||||
|
||||
CORPUS.push(
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Describe these.' },
|
||||
{ type: 'image_url', image_url: PNG },
|
||||
{
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'image/png',
|
||||
data: PNG.split(',')[1],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'video_url',
|
||||
video_url: { url: 'data:video/mp4;base64,AAAA' },
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
file: {
|
||||
filename: 'a.pdf',
|
||||
file_data: 'data:application/pdf;base64,JVBERi0=',
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'application/pdf',
|
||||
data: 'JVBERi0=',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Read it.' },
|
||||
{ puter_path: '~/Documents/a.png' },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
// Model output sent back, in each vendor's shape.
|
||||
{
|
||||
messages: [
|
||||
{ role: 'user', content: 'Think, then search.' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'thinking', thinking: 'Hm.', signature: 'sig' },
|
||||
{ type: 'redacted_thinking', data: 'x' },
|
||||
{
|
||||
type: 'reasoning',
|
||||
id: 'rs_1',
|
||||
summary: [{ type: 'summary_text', text: 'Hm.' }],
|
||||
encrypted_content: 'enc',
|
||||
},
|
||||
{ type: 'text', text: 'Searching.' },
|
||||
{
|
||||
type: 'server_tool_use',
|
||||
id: 'srvtoolu_1',
|
||||
name: 'web_search',
|
||||
input: { query: 'x' },
|
||||
},
|
||||
{
|
||||
type: 'web_search_tool_result',
|
||||
tool_use_id: 'srvtoolu_1',
|
||||
content: [],
|
||||
},
|
||||
{
|
||||
type: 'compaction',
|
||||
id: 'cmp_1',
|
||||
encrypted_content: 'enc',
|
||||
},
|
||||
],
|
||||
reasoning: 'Hm.',
|
||||
refusal: null,
|
||||
},
|
||||
{ type: 'compaction', id: 'cmp_2', encrypted_content: 'enc' },
|
||||
{ role: 'user', content: 'Go on.' },
|
||||
],
|
||||
tools: [
|
||||
{ type: 'web_search_20250305', name: 'web_search', max_uses: 3 },
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'strict_tool',
|
||||
strict: true,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'anthropic_style',
|
||||
description: 'x',
|
||||
input_schema: { type: 'object', properties: {} },
|
||||
},
|
||||
],
|
||||
tool_choice: 'required',
|
||||
thinking: { type: 'adaptive' },
|
||||
prompt_cache_options: { ttl: '30m' },
|
||||
instructions: [{ role: 'system', content: 'Be brief.' }],
|
||||
},
|
||||
);
|
||||
|
||||
const OPTION_NAMES = [
|
||||
...new Set(CORPUS.flatMap((request) => Object.keys(request))),
|
||||
'previous_response_id',
|
||||
'conversation',
|
||||
'prompt',
|
||||
'prompt_cache_options',
|
||||
'background',
|
||||
'custom',
|
||||
'customLimitMessage',
|
||||
'test_mode',
|
||||
];
|
||||
|
||||
// -- Harness --
|
||||
|
||||
const key = { apiKey: 'fuzz-key' };
|
||||
const providers = {
|
||||
claude: key,
|
||||
'azure-openai': { ...key, apiURL: 'https://azure.upstream.test/openai' },
|
||||
'openai-completion': key,
|
||||
gemini: key,
|
||||
meta: key,
|
||||
groq: key,
|
||||
deepseek: key,
|
||||
mistral: key,
|
||||
xai: key,
|
||||
moonshot: key,
|
||||
minimax: key,
|
||||
zai: key,
|
||||
alibaba: key,
|
||||
byteplus: key,
|
||||
hoonify: key,
|
||||
ollama: { enabled: false },
|
||||
};
|
||||
|
||||
let server: PuterServer;
|
||||
let driver: ChatCompletionDriver;
|
||||
// Every provider gets the same share of cases, whatever its catalog size.
|
||||
let routes: Map<string, { provider: string; id: string }[]>;
|
||||
|
||||
beforeAll(async () => {
|
||||
upstream.install();
|
||||
server = await setupTestServer();
|
||||
driver = new ChatCompletionDriver(
|
||||
{ providers } as never,
|
||||
server.clients,
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
await driver.onServerStart();
|
||||
routes = new Map();
|
||||
for (const model of await driver.models()) {
|
||||
if (!model.provider || model.provider === 'fake-chat') continue;
|
||||
const route = { provider: model.provider, id: model.id };
|
||||
routes.set(route.provider, [
|
||||
...(routes.get(route.provider) ?? []),
|
||||
route,
|
||||
]);
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
upstream.uninstall();
|
||||
});
|
||||
|
||||
const pickRoute = (random: Random) =>
|
||||
pickStable(
|
||||
random,
|
||||
routes.get(pickStable(random, [...routes.keys()], (name) => name))!,
|
||||
(route) => route.id,
|
||||
);
|
||||
|
||||
/** Runs one completion to its end, stream included, and returns its failure. */
|
||||
const complete = async (args: Record<string, unknown>): Promise<unknown> => {
|
||||
try {
|
||||
const result = await withTestActor(() =>
|
||||
driver.complete(
|
||||
structuredClone(args) as unknown as ICompleteArguments,
|
||||
),
|
||||
);
|
||||
if (isDriverStreamResult(result)) await readStreamResult(result);
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
return error;
|
||||
}
|
||||
};
|
||||
|
||||
describe('ChatCompletionDriver seeded fuzzing', () => {
|
||||
it('covers every configured provider', () => {
|
||||
expect([...routes.keys()].sort()).toEqual([
|
||||
'alibaba',
|
||||
'azure-openai',
|
||||
'azure-openai-responses',
|
||||
'byteplus',
|
||||
'claude',
|
||||
'deepseek',
|
||||
'gemini',
|
||||
'groq',
|
||||
'hoonify',
|
||||
'meta',
|
||||
'minimax',
|
||||
'mistral',
|
||||
'moonshotai',
|
||||
'openai-completion',
|
||||
'openai-responses',
|
||||
'xai',
|
||||
'zai',
|
||||
]);
|
||||
});
|
||||
|
||||
it('answers or rejects a malformed request without failing a healthy route', async () => {
|
||||
upstream.respond = healthyUpstream;
|
||||
upstream.unanswered.clear();
|
||||
clearUnhealthyRoutes();
|
||||
const findings = new Findings();
|
||||
let answered = 0;
|
||||
|
||||
for (let index = 0; index < fuzzCases(3000); index++) {
|
||||
const random = caseRandom(0xc4a7, index);
|
||||
const route = pickRoute(random);
|
||||
const args: Record<string, unknown> = {
|
||||
...mutate(pick(random, CORPUS), random, OPTION_NAMES),
|
||||
provider: route.provider,
|
||||
model: route.id,
|
||||
};
|
||||
// Now and then the route itself is the malformed part.
|
||||
if (random() < 0.05) {
|
||||
args[pick(random, ['provider', 'model'])] =
|
||||
hostileValue(random);
|
||||
}
|
||||
const label = `${route.provider}:${route.id}`;
|
||||
const error = await complete(args);
|
||||
if (!error) answered++;
|
||||
const unclassified = error && unclassifiedFailure(error);
|
||||
if (unclassified) findings.add(unclassified, index, label, args);
|
||||
else if (isHttpError(error) && error.statusCode >= 500) {
|
||||
const attempts = (error.fields?.attempts ?? []) as {
|
||||
status?: number;
|
||||
error: string;
|
||||
}[];
|
||||
findings.add(
|
||||
`${error.statusCode} ${error.legacyCode}: ${error.message} [${attempts.map((a) => `${a.status ?? 'no status'} ${a.error}`).join(' | ')}]`,
|
||||
index,
|
||||
label,
|
||||
args,
|
||||
);
|
||||
}
|
||||
// The vendors answered every request, so no route failed.
|
||||
for (const marked of kv.keys('aiChat:unhealthyRoute:*')) {
|
||||
findings.add(`marked unhealthy ${marked}`, index, label, args);
|
||||
}
|
||||
clearUnhealthyRoutes();
|
||||
}
|
||||
|
||||
for (const request of upstream.unanswered) {
|
||||
findings.add(`no upstream fixture for ${request}`, -1, '', {});
|
||||
}
|
||||
findings.assertNone();
|
||||
// A run where nothing is answered has stopped reaching the providers.
|
||||
expect(answered).toBeGreaterThan(0);
|
||||
}, 600_000);
|
||||
|
||||
it('ends every request with an answer or a classified error when the upstream misbehaves', async () => {
|
||||
const findings = new Findings();
|
||||
const askedBefore = upstream.asked;
|
||||
|
||||
for (let index = 0; index < fuzzCases(2000); index++) {
|
||||
const random = caseRandom(0x0b5e, index);
|
||||
const route = pickRoute(random);
|
||||
const replies: unknown[] = [];
|
||||
upstream.respond = (request) => {
|
||||
const healthy = healthyUpstream(request);
|
||||
const reply = healthy && corruptReply(healthy, random);
|
||||
replies.push(reply);
|
||||
return reply;
|
||||
};
|
||||
const error = await complete({
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
provider: route.provider,
|
||||
model: route.id,
|
||||
stream: random() < 0.5,
|
||||
fallback: random() < 0.5,
|
||||
});
|
||||
const unclassified = error && unclassifiedFailure(error);
|
||||
if (unclassified) {
|
||||
findings.add(
|
||||
unclassified,
|
||||
index,
|
||||
`${route.provider}:${route.id}`,
|
||||
replies,
|
||||
);
|
||||
}
|
||||
clearUnhealthyRoutes();
|
||||
}
|
||||
findings.assertNone();
|
||||
expect(upstream.asked).toBeGreaterThan(askedBefore);
|
||||
}, 600_000);
|
||||
});
|
||||
+5
-5
@@ -38,11 +38,11 @@ import {
|
||||
vi,
|
||||
} from 'vitest';
|
||||
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { withTestActor } from '../integrationTestUtil.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../shared/integrationTestUtil.js';
|
||||
import { ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import { FakeChatProvider } from './providers/FakeChatProvider.js';
|
||||
import { FakeChatProvider } from '../../providers/fake/chat/FakeChatProvider.js';
|
||||
import type { IChatModel, ICompleteArguments } from './types.js';
|
||||
|
||||
// The puter_path resolver reads the user's filesystem; stub it and assert on
|
||||
@@ -51,7 +51,7 @@ const { processPuterPathUploadsMock } = vi.hoisted(() => ({
|
||||
processPuterPathUploadsMock: vi.fn(async (_messages: unknown) => {}),
|
||||
}));
|
||||
|
||||
vi.mock('./providers/openai/fileUpload.js', () => ({
|
||||
vi.mock('../../providers/openai/chat/fileUpload.js', () => ({
|
||||
processPuterPathUploads: processPuterPathUploadsMock,
|
||||
MAX_FILE_SIZE: 5 * 1_000_000,
|
||||
}));
|
||||
+6
-6
@@ -38,14 +38,14 @@ import {
|
||||
vi,
|
||||
} from 'vitest';
|
||||
|
||||
import { Context } from '../../core/context.js';
|
||||
import type { UsageInput } from '../../services/metering/types.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { withTestActor } from '../integrationTestUtil.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { UsageInput } from '../../../../services/metering/types.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../shared/integrationTestUtil.js';
|
||||
import { ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import { clearUnhealthyRoutes, isRouteUnhealthy } from './utils/providerHealth.js';
|
||||
import { FakeChatProvider } from './providers/FakeChatProvider.js';
|
||||
import { FakeChatProvider } from '../../providers/fake/chat/FakeChatProvider.js';
|
||||
import type { IChatCompleteResult } from './types.js';
|
||||
import { SDK_TIMEOUT_MS } from './utils/sdkClient.js';
|
||||
import type { AIChatStream } from './utils/Streaming.js';
|
||||
+9
-9
@@ -44,21 +44,21 @@ import type { Request } from 'express';
|
||||
import {
|
||||
renderAnthropicError,
|
||||
renderOpenAIError,
|
||||
} from '../../controllers/puterai/wireErrors.js';
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { withTestActor } from '../integrationTestUtil.js';
|
||||
} from '../../../../controllers/puterai/wireErrors.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../shared/integrationTestUtil.js';
|
||||
import { ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import { OPEN_AI_MODELS } from './providers/openai/models.js';
|
||||
import { OPEN_AI_MODELS } from '../../providers/openai/chat/models.js';
|
||||
import {
|
||||
clearUnhealthyRoutes,
|
||||
isRouteUnhealthy,
|
||||
markRouteUnhealthy,
|
||||
} from './utils/providerHealth.js';
|
||||
import { InfronProvider } from './providers/infron/InfronProvider.js';
|
||||
import { UpstreamDeadlineError } from '../util/upstreamErrors.js';
|
||||
import { InfronProvider } from '../../providers/infron/chat/InfronProvider.js';
|
||||
import { UpstreamDeadlineError } from '../../../util/upstreamErrors.js';
|
||||
|
||||
// -- OpenAI SDK mock ------------------------------------------------
|
||||
// Gemini reaches Google through `new openai.OpenAI()` (default export) and
|
||||
+148
-10
@@ -44,17 +44,17 @@ import {
|
||||
vi,
|
||||
} from 'vitest';
|
||||
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import { withTestActor } from '../integrationTestUtil.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { PuterServer } from '../../../../server.js';
|
||||
import { setupTestServer } from '../../../../testUtil.js';
|
||||
import { withTestActor } from '../../shared/integrationTestUtil.js';
|
||||
import { COUNT_TOKENS, ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import { ClaudeProvider } from './providers/claude/ClaudeProvider.js';
|
||||
import { FakeChatProvider } from './providers/FakeChatProvider.js';
|
||||
import { OllamaChatProvider } from './providers/ollama/OllamaProvider.js';
|
||||
import { XAI_CHAT_MODELS } from './providers/xai/catalog.js';
|
||||
import { XAIProvider } from './providers/xai/XAIProvider.js';
|
||||
import { ClaudeProvider } from '../../providers/claude/chat/ClaudeProvider.js';
|
||||
import { FakeChatProvider } from '../../providers/fake/chat/FakeChatProvider.js';
|
||||
import { OllamaChatProvider } from '../../providers/ollama/chat/OllamaProvider.js';
|
||||
import { XAI_CHAT_MODELS } from '../../providers/xai/chat/catalog.js';
|
||||
import { XAIProvider } from '../../providers/xai/chat/XAIProvider.js';
|
||||
import type {
|
||||
IChatCompleteResult,
|
||||
IChatProvider,
|
||||
@@ -207,6 +207,50 @@ describe('ChatCompletionDriver model map', () => {
|
||||
d.onServerShutdown();
|
||||
});
|
||||
|
||||
it('lists qualified model ids without publishing aliases as models', async () => {
|
||||
const ids = [
|
||||
'kimi/kimi-3',
|
||||
'ollama:plain-model',
|
||||
'ollama:vendor/qualified',
|
||||
'vendor/tagged:free',
|
||||
];
|
||||
vi.spyOn(OllamaChatProvider.prototype, 'models').mockResolvedValue(
|
||||
ids.map((id) => ({
|
||||
id,
|
||||
aliases: ['legacy-' + id],
|
||||
costs_currency: 'usd-cents',
|
||||
costs: {},
|
||||
max_tokens: 8192,
|
||||
})),
|
||||
);
|
||||
const d = newDriver({ ollama: {} });
|
||||
await d.onServerStart();
|
||||
try {
|
||||
expect(
|
||||
(await d.list()).filter(
|
||||
(id) => !['abuse', 'costly', 'fake'].includes(id),
|
||||
),
|
||||
).toEqual([
|
||||
'ollama:kimi/kimi-3',
|
||||
'ollama:ollama/plain-model',
|
||||
'ollama:vendor/qualified',
|
||||
'ollama:vendor/tagged:free',
|
||||
]);
|
||||
const models = (await d.models()).filter(
|
||||
(model) => model.provider === 'ollama',
|
||||
);
|
||||
expect(models.map((model) => model.id).sort()).toEqual(
|
||||
[...ids].sort(),
|
||||
);
|
||||
for (const model of models) {
|
||||
expect(model.aliases).toContain(model.puterId);
|
||||
expect(model.aliases).toContain('legacy-' + model.id);
|
||||
}
|
||||
} finally {
|
||||
d.onServerShutdown();
|
||||
}
|
||||
});
|
||||
|
||||
it('lists the catalog to a caller that asks while it is still loading', async () => {
|
||||
const d = newDriver({ ollama: { enabled: false } });
|
||||
d.onServerStart();
|
||||
@@ -2396,3 +2440,97 @@ describe('ChatCompletionDriver service speed', () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('ChatCompletionDriver.complete option shapes', () => {
|
||||
const hi = [{ role: 'user', content: 'hi' }];
|
||||
|
||||
it.each(['model', 'provider', 'service_tier'])(
|
||||
'rejects a value that cannot be text as `%s`',
|
||||
async (name) => {
|
||||
const args = { model: 'fake', messages: hi, [name]: { toString: null } };
|
||||
await expect(
|
||||
withTestActor(() => driver.complete(args as never)),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
legacyCode: 'bad_request',
|
||||
message: `\`${name}\` is not a usable value`,
|
||||
});
|
||||
await expect(
|
||||
withTestActor(() => driver[COUNT_TOKENS](args as never)),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
},
|
||||
);
|
||||
|
||||
it('routes by the model when `provider` is an object, as it did before', async () => {
|
||||
const completeSpy = vi.spyOn(FakeChatProvider.prototype, 'complete');
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: 'fake',
|
||||
messages: hi,
|
||||
provider: { order: ['Anthropic'] },
|
||||
} as never),
|
||||
);
|
||||
expect(completeSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([false, 0, '', null])(
|
||||
'treats `tools: %j` as no tools when counting tokens',
|
||||
async (tools) => {
|
||||
const countTokens = vi.fn(async (_args: object) => 1);
|
||||
const fake = FakeChatProvider.prototype as {
|
||||
countTokens?: typeof countTokens;
|
||||
};
|
||||
fake.countTokens = countTokens;
|
||||
try {
|
||||
await withTestActor(() =>
|
||||
driver[COUNT_TOKENS]({
|
||||
model: 'fake',
|
||||
messages: hi,
|
||||
tools,
|
||||
} as never),
|
||||
);
|
||||
} finally {
|
||||
delete fake.countTokens;
|
||||
}
|
||||
expect(countTokens.mock.calls[0]![0]).not.toHaveProperty('tools');
|
||||
},
|
||||
);
|
||||
|
||||
it.each([false, 0, '', null])(
|
||||
'treats `tools: %j` as no tools',
|
||||
async (tools) => {
|
||||
const completeSpy = vi.spyOn(FakeChatProvider.prototype, 'complete');
|
||||
await withTestActor(() =>
|
||||
driver.complete({ model: 'fake', messages: hi, tools } as never),
|
||||
);
|
||||
expect(completeSpy.mock.calls[0]![0]).not.toHaveProperty('tools');
|
||||
},
|
||||
);
|
||||
|
||||
it('reports zero tokens in the cost event for a usage count that is not a number', async () => {
|
||||
vi.spyOn(FakeChatProvider.prototype, 'models').mockResolvedValueOnce([
|
||||
{
|
||||
id: 'priced',
|
||||
aliases: [],
|
||||
costs_currency: 'usd-cents',
|
||||
costs: { input_tokens: 1000, output_tokens: 2000 },
|
||||
max_tokens: 8192,
|
||||
},
|
||||
]);
|
||||
const d = await makeDriver();
|
||||
const costEvents = captureEvent('ai.prompt.cost-calculated');
|
||||
vi.spyOn(FakeChatProvider.prototype, 'complete').mockResolvedValueOnce({
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'ok' }] },
|
||||
usage: { input_tokens: 10, output_tokens: { toString: null } },
|
||||
finish_reason: 'stop',
|
||||
} as never);
|
||||
|
||||
await withTestActor(() => d.complete({ model: 'priced', messages: hi }));
|
||||
|
||||
expect(costEvents[0]![1]).toMatchObject({
|
||||
input_tokens: 10,
|
||||
output_tokens: 0,
|
||||
total_ucents: 10 * 1000,
|
||||
});
|
||||
});
|
||||
});
|
||||
+108
-58
@@ -19,56 +19,56 @@
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import { EventMap } from '../../clients/event/types.js';
|
||||
import type { Actor } from '../../core/actor.js';
|
||||
import { Context } from '../../core/context.js';
|
||||
import { HttpError, isHttpError } from '../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../services/metering/enforcement.js';
|
||||
import { isFreeSubscription } from '../../services/metering/consts.js';
|
||||
import type { MeteringService } from '../../services/metering/MeteringService.js';
|
||||
import type { CreditHold } from '../../services/metering/types.js';
|
||||
import { NO_CREDIT_HOLD } from '../../services/metering/types.js';
|
||||
import type { DriverStreamResult } from '../meta.js';
|
||||
import { PuterDriver } from '../types.js';
|
||||
import { EventMap } from '../../../../clients/event/types.js';
|
||||
import type { Actor } from '../../../../core/actor.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import { HttpError, isHttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { isFreeSubscription } from '../../../../services/metering/consts.js';
|
||||
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
|
||||
import type { CreditHold } from '../../../../services/metering/types.js';
|
||||
import { NO_CREDIT_HOLD } from '../../../../services/metering/types.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import { PuterDriver } from '../../../types.js';
|
||||
import {
|
||||
type AiMeteringService,
|
||||
factoredTotal,
|
||||
requestAiCosts,
|
||||
withAiCostFactor,
|
||||
} from '../util/aiCostFactor.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js';
|
||||
} from '../../../../util/aiCostFactor.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../../shared/aiLimits.js';
|
||||
import {
|
||||
isCreditExhaustion as isUpstreamCreditExhaustion,
|
||||
isUpstreamConnectionError,
|
||||
isUpstreamTimeoutError,
|
||||
sanitizeUpstreamMessage,
|
||||
UpstreamDeadlineError,
|
||||
} from '../util/upstreamErrors.js';
|
||||
import { AlibabaProvider } from './providers/alibaba/AlibabaProvider.js';
|
||||
import { AzureChatProvider } from './providers/azure/AzureChatProvider.js';
|
||||
import { AzureResponsesProvider } from './providers/azure/AzureResponsesProvider.js';
|
||||
import { BytePlusProvider } from './providers/byteplus/BytePlusProvider.js';
|
||||
import { ClaudeProvider } from './providers/claude/ClaudeProvider.js';
|
||||
import { DeepSeekProvider } from './providers/deepseek/DeepSeekProvider.js';
|
||||
import { FakeChatProvider } from './providers/FakeChatProvider.js';
|
||||
import { GeminiChatProvider } from './providers/gemini/GeminiChatProvider.js';
|
||||
import { GroqAIProvider } from './providers/groq/GroqAIProvider.js';
|
||||
import { HoonifyProvider } from './providers/hoonify/HoonifyProvider.js';
|
||||
import { InfronProvider } from './providers/infron/InfronProvider.js';
|
||||
import { MetaProvider } from './providers/meta/MetaProvider.js';
|
||||
import { MiniMaxProvider } from './providers/minimax/MiniMaxProvider.js';
|
||||
import { MistralAIProvider } from './providers/mistral/MistralAiProvider.js';
|
||||
import { MoonshotProvider } from './providers/moonshot/MoonshotProvider.js';
|
||||
import { NeuralwattProvider } from './providers/neuralwatt/NeuralwattProvider.js';
|
||||
import { OllamaChatProvider } from './providers/ollama/OllamaProvider.js';
|
||||
import type { ChatProviderConfig } from './providers/OpenAICompatProvider.js';
|
||||
import { processPuterPathUploads } from './providers/openai/fileUpload.js';
|
||||
import { OpenAiChatProvider } from './providers/openai/OpenAiChatCompletionsProvider.js';
|
||||
import { OpenAiResponsesChatProvider } from './providers/openai/OpenAiChatResponsesProvider.js';
|
||||
import { OpenRouterProvider } from './providers/openrouter/OpenRouterProvider.js';
|
||||
import { TogetherAIProvider } from './providers/together/TogetherAIProvider.js';
|
||||
import { XAIProvider } from './providers/xai/XAIProvider.js';
|
||||
import { ZAIProvider } from './providers/zai/ZAIProvider.js';
|
||||
} from '../../../util/upstreamErrors.js';
|
||||
import { AlibabaProvider } from '../../providers/alibaba/chat/AlibabaProvider.js';
|
||||
import { AzureChatProvider } from '../../providers/azure/chat/AzureChatProvider.js';
|
||||
import { AzureResponsesProvider } from '../../providers/azure/chat/AzureResponsesProvider.js';
|
||||
import { BytePlusProvider } from '../../providers/byteplus/chat/BytePlusProvider.js';
|
||||
import { ClaudeProvider } from '../../providers/claude/chat/ClaudeProvider.js';
|
||||
import { DeepSeekProvider } from '../../providers/deepseek/chat/DeepSeekProvider.js';
|
||||
import { FakeChatProvider } from '../../providers/fake/chat/FakeChatProvider.js';
|
||||
import { GeminiChatProvider } from '../../providers/gemini/chat/GeminiChatProvider.js';
|
||||
import { GroqAIProvider } from '../../providers/groq/chat/GroqAIProvider.js';
|
||||
import { HoonifyProvider } from '../../providers/hoonify/chat/HoonifyProvider.js';
|
||||
import { InfronProvider } from '../../providers/infron/chat/InfronProvider.js';
|
||||
import { MetaProvider } from '../../providers/meta/chat/MetaProvider.js';
|
||||
import { MiniMaxProvider } from '../../providers/minimax/chat/MiniMaxProvider.js';
|
||||
import { MistralAIProvider } from '../../providers/mistral/chat/MistralAiProvider.js';
|
||||
import { MoonshotProvider } from '../../providers/moonshot/chat/MoonshotProvider.js';
|
||||
import { NeuralwattProvider } from '../../providers/neuralwatt/chat/NeuralwattProvider.js';
|
||||
import { OllamaChatProvider } from '../../providers/ollama/chat/OllamaProvider.js';
|
||||
import type { ChatProviderConfig } from '../../protocols/openaiCompatible/OpenAICompatProvider.js';
|
||||
import { processPuterPathUploads } from '../../providers/openai/chat/fileUpload.js';
|
||||
import { OpenAiChatProvider } from '../../providers/openai/chat/OpenAiChatCompletionsProvider.js';
|
||||
import { OpenAiResponsesChatProvider } from '../../providers/openai/chat/OpenAiChatResponsesProvider.js';
|
||||
import { OpenRouterProvider } from '../../providers/openrouter/chat/OpenRouterProvider.js';
|
||||
import { TogetherAIProvider } from '../../providers/together/chat/TogetherAIProvider.js';
|
||||
import { XAIProvider } from '../../providers/xai/chat/XAIProvider.js';
|
||||
import { ZAIProvider } from '../../providers/zai/chat/ZAIProvider.js';
|
||||
import type {
|
||||
IChatCompleteResult,
|
||||
IChatMessageResult,
|
||||
@@ -103,9 +103,13 @@ import {
|
||||
normalizeResultToOpenAI,
|
||||
shouldPresentAsOpenAI,
|
||||
} from './utils/normalizeToOpenAI.js';
|
||||
import { isToolChoice, toolChoiceFromWire } from './utils/openaiParams.js';
|
||||
import {
|
||||
catalogPrice,
|
||||
isToolChoice,
|
||||
toolChoiceFromWire,
|
||||
} from '../../protocols/openaiCompatible/openaiParams.js';
|
||||
import { assertScalarOptions } from '../../shared/scalarOptions.js';
|
||||
import { catalogPrice } from '../../shared/catalogPrice.js';
|
||||
import {
|
||||
costKeys,
|
||||
costsForRequest,
|
||||
isFreeModel,
|
||||
@@ -121,7 +125,7 @@ import {
|
||||
TRANSIENT_UNHEALTHY_TTL_SEC,
|
||||
UNHEALTHY_TTL_SEC,
|
||||
} from './utils/providerHealth.js';
|
||||
import { CATALOG_TTL_MS } from './utils/cachedRemoteCatalog.js';
|
||||
import { CATALOG_TTL_MS } from '../../shared/cachedRemoteCatalog.js';
|
||||
import { SDK_TIMEOUT_MS } from './utils/sdkClient.js';
|
||||
import { fromFinishReason } from './utils/stopReason.js';
|
||||
import { AIChatStream } from './utils/Streaming.js';
|
||||
@@ -178,6 +182,9 @@ export const COUNT_TOKENS = Symbol('aiChat.countTokens');
|
||||
*/
|
||||
const HOLD_RENEW_INTERVAL_MS = 5 * 60 * 1000;
|
||||
|
||||
// Options that reach error messages and request headers as text.
|
||||
const SCALAR_OPTIONS = ['model', 'provider', 'service_tier'];
|
||||
|
||||
/**
|
||||
* A moderation refusal is a completion that was produced and charged, then
|
||||
* withheld — not a route failure. Retrying it on a fallback provider would bill
|
||||
@@ -585,7 +592,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
readonly driverName = 'ai-chat';
|
||||
readonly isDefault = true;
|
||||
|
||||
// Shared AI policy — see `drivers/util/aiLimits.ts` for the tier table.
|
||||
// Shared AI policy — see `drivers/ai/shared/aiLimits.ts` for the tier table.
|
||||
readonly rateLimit = AI_RATE_LIMIT;
|
||||
readonly concurrent = AI_CONCURRENT;
|
||||
|
||||
@@ -690,6 +697,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
{ legacyCode: 'invalid_service_speed' },
|
||||
);
|
||||
}
|
||||
assertScalarOptions(args, SCALAR_OPTIONS);
|
||||
const abortSignal = Context.get('abortSignal');
|
||||
abortSignal?.throwIfAborted();
|
||||
const actor = Context.get('actor');
|
||||
@@ -744,6 +752,9 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
args.messages = normalizeMediaParts(normalize_messages(args.messages));
|
||||
if (args.tools) {
|
||||
normalize_tools_object(args.tools);
|
||||
} else {
|
||||
// `tools: enabled && list` leaves `false` here; it means no tools.
|
||||
delete args.tools;
|
||||
}
|
||||
// A raw `/drivers/call` caller still sends the OpenAI wire form
|
||||
// ('auto'/'none'/'required', {type:'function', function:{name}}) —
|
||||
@@ -1297,8 +1308,17 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
}, HOLD_RENEW_INTERVAL_MS);
|
||||
renewHold.unref?.();
|
||||
|
||||
// Fire-and-forget — the stream writes happen async while the
|
||||
// response is being piped to the client.
|
||||
// The upstream runs on after a caller hangs up, so the request
|
||||
// keeps its concurrency slot until the upstream is done, not until
|
||||
// the cleanup after it.
|
||||
let upstreamDone!: () => void;
|
||||
const upstreamRunning = new Promise<void>((resolve) => {
|
||||
upstreamDone = resolve;
|
||||
});
|
||||
Context.get('keepConcurrencySlot')?.(upstreamRunning);
|
||||
|
||||
// The stream writes happen async while the response is piped to
|
||||
// the client.
|
||||
(async () => {
|
||||
try {
|
||||
await init({ chatStream });
|
||||
@@ -1323,6 +1343,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
}
|
||||
passthrough.end();
|
||||
} finally {
|
||||
upstreamDone();
|
||||
clearInterval(renewHold);
|
||||
clearTimeout(detachedLimit);
|
||||
abortSignal?.removeEventListener('abort', detachStream);
|
||||
@@ -1490,6 +1511,7 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
async [COUNT_TOKENS](
|
||||
args: ICompleteArguments,
|
||||
): Promise<{ input_tokens: number }> {
|
||||
assertScalarOptions(args, SCALAR_OPTIONS);
|
||||
let intendedProvider = args.provider || '';
|
||||
if (!args.model && !intendedProvider) {
|
||||
intendedProvider = 'azure-openai';
|
||||
@@ -1515,6 +1537,9 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
args.messages = normalizeMediaParts(normalize_messages(args.messages));
|
||||
if (args.tools) {
|
||||
normalize_tools_object(args.tools);
|
||||
} else {
|
||||
// `tools: enabled && list` leaves `false` here; it means no tools.
|
||||
delete args.tools;
|
||||
}
|
||||
|
||||
const provider = this.#providers[model.provider!];
|
||||
@@ -1848,20 +1873,20 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
|
||||
const cost = usageCost(usage, model, usageCosts, pricingTime);
|
||||
const { inputKey, outputKey } = costKeys(model);
|
||||
// The vendor reports usage, and pricing skips a count that is not a
|
||||
// number; so does this.
|
||||
const count = (value: unknown) =>
|
||||
typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||
const inputTokens = cost
|
||||
? Number(
|
||||
usage[inputKey] ??
|
||||
usage.prompt_tokens ??
|
||||
usage.input_tokens ??
|
||||
0,
|
||||
? count(
|
||||
usage[inputKey] ?? usage.prompt_tokens ?? usage.input_tokens,
|
||||
)
|
||||
: 0;
|
||||
const outputTokens = cost
|
||||
? Number(
|
||||
? count(
|
||||
usage[outputKey] ??
|
||||
usage.completion_tokens ??
|
||||
usage.output_tokens ??
|
||||
0,
|
||||
usage.output_tokens,
|
||||
)
|
||||
: 0;
|
||||
const inputMicroCents = cost?.inputMicroCents ?? 0;
|
||||
@@ -2061,14 +2086,39 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
// instance, so they are read and never written: normalizing
|
||||
// the id or appending puterId in place would accumulate across
|
||||
// instantiations. The bucket gets its own copy instead.
|
||||
const aliases =
|
||||
entry.puterId &&
|
||||
!(entry.aliases ?? []).includes(entry.puterId)
|
||||
? [...(entry.aliases ?? []), entry.puterId]
|
||||
: entry.aliases;
|
||||
// Keep upstream ids for provider calls and publish a qualified
|
||||
// routing name even when a catalog omits its vendor namespace.
|
||||
const publicId = entry.puterId || entry.id;
|
||||
// A colon after the first slash belongs to the model name, as
|
||||
// in `vendor/model:free`, and names no service.
|
||||
const serviceSeparator = publicId
|
||||
.split('/', 1)[0]!
|
||||
.indexOf(':');
|
||||
const service =
|
||||
serviceSeparator < 0
|
||||
? providerName
|
||||
: publicId.slice(0, serviceSeparator);
|
||||
const modelPath =
|
||||
serviceSeparator < 0
|
||||
? publicId
|
||||
: publicId.slice(serviceSeparator + 1);
|
||||
const puterId =
|
||||
providerName === 'fake-chat'
|
||||
? entry.puterId
|
||||
: `${service}:${modelPath.includes('/') ? modelPath : `${service}/${modelPath}`}`;
|
||||
const aliases = puterId
|
||||
? [
|
||||
...new Set([
|
||||
...(entry.aliases ?? []),
|
||||
...(entry.puterId ? [entry.puterId] : []),
|
||||
puterId,
|
||||
]),
|
||||
]
|
||||
: entry.aliases;
|
||||
const model = {
|
||||
...entry,
|
||||
id: normalizeModelKey(entry.id),
|
||||
...(puterId ? { puterId } : {}),
|
||||
};
|
||||
// Assigned only when the entry has names to carry: models()
|
||||
// is serialized to the API, and an entry that declared no
|
||||
+42
-31
@@ -47,29 +47,29 @@
|
||||
* on that test.)
|
||||
*/
|
||||
|
||||
import { readdirSync } from 'node:fs';
|
||||
import { existsSync, readdirSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import type { IChatModel } from '../types.js';
|
||||
import { ALIBABA_MODELS } from './alibaba/models.js';
|
||||
import { AZURE_MODELS } from './azure/models.js';
|
||||
import { BYTEPLUS_MODELS } from './byteplus/models.js';
|
||||
import { CLAUDE_MODELS } from './claude/models.js';
|
||||
import { DEEPSEEK_MODELS } from './deepseek/models.js';
|
||||
import { GEMINI_MODELS } from './gemini/models.js';
|
||||
import { GROQ_MODELS } from './groq/models.js';
|
||||
import { HOONIFY_MODELS } from './hoonify/models.js';
|
||||
import { META_MODELS } from './meta/models.js';
|
||||
import { MINIMAX_MODELS } from './minimax/models.js';
|
||||
import { MISTRAL_MODELS } from './mistral/models.js';
|
||||
import { MOONSHOT_MODELS } from './moonshot/models.js';
|
||||
import { OPEN_AI_MODELS } from './openai/models.js';
|
||||
import { OPEN_ROUTER_MODEL_OVERRIDES } from './openrouter/modelOverrides.js';
|
||||
import { XAI_MODELS } from './xai/models.js';
|
||||
import { ZAI_MODELS } from './zai/models.js';
|
||||
import type { IChatModel } from './types.js';
|
||||
import { ALIBABA_MODELS } from '../../providers/alibaba/chat/models.js';
|
||||
import { AZURE_MODELS } from '../../providers/azure/chat/models.js';
|
||||
import { BYTEPLUS_MODELS } from '../../providers/byteplus/chat/models.js';
|
||||
import { CLAUDE_MODELS } from '../../providers/claude/chat/models.js';
|
||||
import { DEEPSEEK_MODELS } from '../../providers/deepseek/chat/models.js';
|
||||
import { GEMINI_MODELS } from '../../providers/gemini/chat/models.js';
|
||||
import { GROQ_MODELS } from '../../providers/groq/chat/models.js';
|
||||
import { HOONIFY_MODELS } from '../../providers/hoonify/chat/models.js';
|
||||
import { META_MODELS } from '../../providers/meta/chat/models.js';
|
||||
import { MINIMAX_MODELS } from '../../providers/minimax/chat/models.js';
|
||||
import { MISTRAL_MODELS } from '../../providers/mistral/chat/models.js';
|
||||
import { MOONSHOT_MODELS } from '../../providers/moonshot/chat/models.js';
|
||||
import { OPEN_AI_MODELS } from '../../providers/openai/chat/models.js';
|
||||
import { OPEN_ROUTER_MODEL_OVERRIDES } from '../../providers/openrouter/chat/modelOverrides.js';
|
||||
import { XAI_MODELS } from '../../providers/xai/chat/models.js';
|
||||
import { ZAI_MODELS } from '../../providers/zai/chat/models.js';
|
||||
|
||||
// Providers whose catalog is fetched at runtime (OpenRouter, Ollama, Together,
|
||||
// Infron, Neuralwatt) have nothing static to check and are absent by design;
|
||||
@@ -201,28 +201,34 @@ describe('CATALOGS', () => {
|
||||
// silent gap these tests are about, so the list is checked against what
|
||||
// is actually on disk.
|
||||
//
|
||||
// The scan covers every non-test source file under providers/*/ with
|
||||
// "model" in its name — models.ts, but also siblings like openrouter's
|
||||
// modelOverrides.ts. Importing every provider file regardless of name
|
||||
// would drag in SDK modules for a filename sweep, so that naming
|
||||
// convention is the one assumption left unenforced here: a catalog in a
|
||||
// file named without "model" would escape this net.
|
||||
it('lists every static catalog under providers/', async () => {
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
// The scan covers every non-test source file under providers/*/chat/
|
||||
// with "model" in its name — models.ts, but also siblings like
|
||||
// openrouter's modelOverrides.ts. Importing every provider file
|
||||
// regardless of name would drag in SDK modules for a filename sweep, so
|
||||
// that naming convention is the one assumption left unenforced here: a
|
||||
// catalog in a file named without "model" would escape this net.
|
||||
it('lists every static catalog under providers/*/chat/', async () => {
|
||||
const providersDir = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
'../../providers',
|
||||
);
|
||||
const registered = new Map(CATALOGS);
|
||||
const found = new Set<string>();
|
||||
const problems: string[] = [];
|
||||
|
||||
for (const dir of readdirSync(here, { withFileTypes: true })) {
|
||||
for (const dir of readdirSync(providersDir, { withFileTypes: true })) {
|
||||
if (!dir.isDirectory()) continue;
|
||||
const chatDir = join(providersDir, dir.name, 'chat');
|
||||
if (!existsSync(chatDir)) continue;
|
||||
|
||||
for (const file of readdirSync(join(here, dir.name))) {
|
||||
for (const file of readdirSync(chatDir)) {
|
||||
if (!/model/i.test(file)) continue;
|
||||
if (!file.endsWith('.ts') || file.endsWith('.test.ts')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const module = await import(
|
||||
pathToFileURL(join(here, dir.name, file)).href
|
||||
pathToFileURL(join(chatDir, file)).href
|
||||
);
|
||||
for (const [name, value] of Object.entries(module)) {
|
||||
// A catalog is a non-empty array of entries carrying an
|
||||
@@ -233,16 +239,19 @@ describe('CATALOGS', () => {
|
||||
value.length > 0 &&
|
||||
typeof value[0]?.id === 'string';
|
||||
if (!isCatalog) continue;
|
||||
found.add(name);
|
||||
|
||||
if (!registered.has(name)) {
|
||||
problems.push(`${dir.name}/${file} exports ${name}`);
|
||||
problems.push(
|
||||
`${dir.name}/chat/${file} exports ${name}`,
|
||||
);
|
||||
} else if (registered.get(name) !== value) {
|
||||
// The row's label names this export but its value is
|
||||
// a different array — the wrong catalog would be the
|
||||
// one getting checked.
|
||||
problems.push(
|
||||
`the CATALOGS row named ${name} does not hold ` +
|
||||
`the ${name} that ${dir.name}/${file} exports`,
|
||||
`the ${name} that ${dir.name}/chat/${file} exports`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -250,5 +259,7 @@ describe('CATALOGS', () => {
|
||||
}
|
||||
|
||||
expect(problems, problems.join('\n')).toEqual([]);
|
||||
// A scan pointed at the wrong directory finds nothing and passes.
|
||||
expect([...found].sort()).toEqual([...registered.keys()].sort());
|
||||
});
|
||||
});
|
||||
Loaded 100 of 716 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user