mirror of
https://github.com/HeyPuter/puter.git
synced 2026-11-13 20:12:57 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
95bceb1e34 |
No files matched your search
+2
-7
@@ -2,9 +2,9 @@
|
||||
# docker-compose.full.yml up -d`. None of the defaults below are safe for
|
||||
# anything beyond a local laptop test.
|
||||
|
||||
# ── Public-facing ports (Caddy) ---------------------------------------
|
||||
# ── Public-facing ports (nginx) ---------------------------------------
|
||||
HTTP_PORT=80
|
||||
# HTTPS_PORT=443 # uncomment after you enable TLS in caddy/Caddyfile
|
||||
# HTTPS_PORT=443 # uncomment after you enable TLS in nginx/nginx.conf
|
||||
|
||||
# ── MariaDB ------------------------------------------------------------
|
||||
MARIADB_ROOT_PASSWORD=replace-with-strong-password
|
||||
@@ -16,8 +16,3 @@ MARIADB_PASSWORD=replace-with-strong-password
|
||||
S3_ACCESS_KEY=puter
|
||||
S3_SECRET_KEY=replace-with-strong-secret
|
||||
S3_BUCKET=puter-local
|
||||
|
||||
# ── Inbound SMTP (compose profile `smtp`) ------------------------------
|
||||
# Host port mapped to the receiver's 2525. 25 is what an MX record means;
|
||||
# many hosting providers block it.
|
||||
SMTP_PORT=25
|
||||
@@ -1,72 +0,0 @@
|
||||
# Default owner, including backend and SDK areas without a specialist below.
|
||||
|
||||
# The last matching rule takes precedence.
|
||||
|
||||
- @Salazareo
|
||||
|
||||
# GUI
|
||||
|
||||
/src/gui/ @jelveh @Salazareo
|
||||
|
||||
# Documentation, developer site, and CLI
|
||||
|
||||
/doc/ @reynaldichernando @Salazareo
|
||||
/src/docs/ @reynaldichernando @Salazareo
|
||||
/src/dev-center/ @reynaldichernando @Salazareo
|
||||
/src/cli/ @reynaldichernando @Salazareo
|
||||
|
||||
# Backend AI
|
||||
|
||||
/src/backend/controllers/puterai/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/ai-\*/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# Backend: AI, hosting, workers, networking, and WebDAV
|
||||
|
||||
/src/backend/controllers/hosting/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/webdav/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/wisp/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/util/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/workers/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/services/localworker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/stores/subdomain/ @ProgrammerIn-wonderland @Salazareo
|
||||
/extensions/workerSandbox\* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker-types/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/peer/ @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# Backend: sharing, permissions, and teams
|
||||
|
||||
/src/backend/controllers/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/controllers/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/acl/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/apps/AppPermissionService\* @jfcastro92 @Salazareo
|
||||
/src/backend/stores/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/group/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/team/ @jfcastro92 @Salazareo
|
||||
|
||||
# SDK: AI, hosting, workers, and networking
|
||||
|
||||
/src/puter-js/src/modules/ai/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/hosting/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Workers* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/networking/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/lib/networkUtils* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/ai.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/hosting.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/workers.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/net.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Peer\* @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# SDK: sharing and permissions
|
||||
|
||||
/src/puter-js/src/modules/perms/ @jfcastro92
|
||||
/src/puter-js/src/modules/FileSystem/operations/share* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/unshare* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/getShares* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/listShared* @jfcastro92 @Salazareo
|
||||
/src/puter-js/tests/api/suites/perms.suite.ts @jfcastro92 @Salazareo
|
||||
/src/puter-js/tests/api/suites/sharing.suite.ts @jfcastro92 @Salazareo
|
||||
@@ -1,20 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 10
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types: ["version-update:semver-major", "version-update:semver-minor"]
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
@@ -1,106 +0,0 @@
|
||||
# This file is machine-generated by `gh actions-lock`.
|
||||
# Do not edit by hand; run `gh actions-lock` to update.
|
||||
# Docs: https://gh.io/actions-lockfile
|
||||
version: 'v0.0.3'
|
||||
workflows:
|
||||
'.github/workflows/ai-provider-integration-tests.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
'.github/workflows/backend-tests.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'davelosert/[email protected]'
|
||||
- 'dorny/[email protected]'
|
||||
'.github/workflows/dependabot-automerge.yml':
|
||||
- 'dependabot/[email protected]'
|
||||
'.github/workflows/docker-image.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
- 'peter-evans/[email protected]'
|
||||
'.github/workflows/docs-build.yml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
'.github/workflows/notify-dev.yaml': []
|
||||
'.github/workflows/notify-prod.yaml': []
|
||||
'.github/workflows/puterjs-tests.yaml':
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'actions/[email protected]'
|
||||
- 'davelosert/[email protected]'
|
||||
- 'dorny/[email protected]'
|
||||
'.github/workflows/tag-alias.yaml':
|
||||
- 'docker/[email protected]'
|
||||
- 'docker/[email protected]'
|
||||
dependencies:
|
||||
'actions/[email protected]':
|
||||
ref: 'v7.0.1'
|
||||
commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
|
||||
owner_id: 44036562
|
||||
repo_id: 197814629
|
||||
'actions/[email protected]':
|
||||
ref: 'v8.0.2'
|
||||
commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
|
||||
owner_id: 44036562
|
||||
repo_id: 192626254
|
||||
'actions/[email protected]':
|
||||
ref: 'v7.1.0'
|
||||
commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
|
||||
owner_id: 44036562
|
||||
repo_id: 189476904
|
||||
'actions/[email protected]':
|
||||
ref: 'v7.0.2'
|
||||
commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
|
||||
owner_id: 44036562
|
||||
repo_id: 192625955
|
||||
'davelosert/[email protected]':
|
||||
ref: 'v2.13.2'
|
||||
commit: 'sha1-74c2d9210b4fc44c84d8213f61d7ec99837b7aa0'
|
||||
owner_id: 4287128
|
||||
repo_id: 501176456
|
||||
'dependabot/[email protected]':
|
||||
ref: 'v3.1.0'
|
||||
commit: 'sha1-25dd0e34f4fe68f24cc83900b1fe3fe149efef98'
|
||||
owner_id: 27347476
|
||||
repo_id: 371068214
|
||||
'docker/[email protected]':
|
||||
ref: 'v7.4.0'
|
||||
commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc'
|
||||
owner_id: 5429470
|
||||
repo_id: 241092383
|
||||
'docker/[email protected]':
|
||||
ref: 'v4.8.0'
|
||||
commit: 'sha1-8aa5831fb8d48d8f76594c7ddde61ae58019f550'
|
||||
owner_id: 5429470
|
||||
repo_id: 287743349
|
||||
'docker/[email protected]':
|
||||
ref: 'v6.2.0'
|
||||
commit: 'sha1-dc802804100637a589fabce1cb79ff13a1411302'
|
||||
owner_id: 5429470
|
||||
repo_id: 306769011
|
||||
'docker/[email protected]':
|
||||
ref: 'v4.4.1'
|
||||
commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
|
||||
owner_id: 5429470
|
||||
repo_id: 288485773
|
||||
'docker/[email protected]':
|
||||
ref: 'v4.4.0'
|
||||
commit: 'sha1-99012661954931238ded8c8b007157a8430204e1'
|
||||
owner_id: 5429470
|
||||
repo_id: 288487894
|
||||
'dorny/[email protected]':
|
||||
ref: 'v4.0.3'
|
||||
commit: 'sha1-ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d'
|
||||
owner_id: 1588572
|
||||
repo_id: 265601040
|
||||
'peter-evans/[email protected]':
|
||||
ref: 'v5.0.0'
|
||||
commit: 'sha1-1b9a80c056b620d92cedb9d9b5a223409c68ddfa'
|
||||
owner_id: 18365890
|
||||
repo_id: 193685475
|
||||
@@ -17,14 +17,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7.1.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
@@ -49,13 +47,3 @@ jobs:
|
||||
PUTER_TEST_AI_MOONSHOT_API_KEY: ${{ secrets.PUTER_TEST_AI_MOONSHOT_API_KEY }}
|
||||
PUTER_TEST_AI_ZAI_API_KEY: ${{ secrets.PUTER_TEST_AI_ZAI_API_KEY }}
|
||||
PUTER_TEST_AI_ELEVENLABS_API_KEY: ${{ secrets.PUTER_TEST_AI_ELEVENLABS_API_KEY }}
|
||||
|
||||
# Realtime conversations through the SDK against live OpenAI. Builds
|
||||
# the SDK bundle these tests load itself, so it runs even when the
|
||||
# step above fails; the tests skip themselves without the key.
|
||||
- name: Run live realtime conversation tests
|
||||
if: ${{ !cancelled() }}
|
||||
run: npm run setupExtensions && (cd src/puter-js && npm run build) && npx vitest run --config src/puter-js/tests/api/vitest.config.ts src/puter-js/tests/api/realtime/realtimeLive.test.ts
|
||||
env:
|
||||
CI: 'true'
|
||||
PUTER_TEST_AI_OPENAI_API_KEY: ${{ secrets.PUTER_TEST_AI_OPENAI_API_KEY }}
|
||||
@@ -1,217 +1,77 @@
|
||||
name: Backend Tests
|
||||
|
||||
# Always triggers on PRs so it can be a required check; the `changes` job below
|
||||
# decides whether backend/extension paths actually changed (test-only changes
|
||||
# match the same globs, since tests are co-located with the code), and `test`
|
||||
# still reports success when they didn't. Pushes to main produce the coverage
|
||||
# PRs compare against. puter.js SDK changes are covered by puterjs-tests.yaml.
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
# Reads main's coverage artifact for the PR comparison.
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
backend: ${{ steps.filter.outputs.backend }}
|
||||
steps:
|
||||
- uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dorny/[email protected]
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
- 'src/backend/**'
|
||||
- 'extensions/**'
|
||||
- 'tools/**'
|
||||
- 'package.json'
|
||||
- 'package-lock.json'
|
||||
- 'tsconfig.json'
|
||||
- 'tsconfig.build.json'
|
||||
- '.github/workflows/backend-tests.yaml'
|
||||
|
||||
# `tsconfig.build.json` builds with `noCheck: true`, so nothing else in CI
|
||||
# runs the type checker and a missing export compiles to `undefined`,
|
||||
# surfacing only when the call is finally reached at runtime. This diffs
|
||||
# against tools/typecheck-baseline.json and fails only on *new* errors.
|
||||
#
|
||||
# The consuming repo runs an equivalent gate, but only once someone bumps the
|
||||
# submodule pointer — too late to keep the error off this repo's default
|
||||
# branch. Hence a gate here, on this repo's own pull requests.
|
||||
typecheck:
|
||||
needs: changes
|
||||
if: github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check (new errors only)
|
||||
run: npm run typecheck
|
||||
|
||||
# The suite is CPU-bound, so it's split across runners. Each shard writes a
|
||||
# blob report (results plus coverage) that the `coverage` job merges.
|
||||
test-shard:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
name: test shard ${{ matrix.shard }}/${{ strategy.job-total }}
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
include:
|
||||
- ref: ${{ github.base_ref }}
|
||||
artifact: base
|
||||
- ref: ${{ github.head_ref }}
|
||||
artifact: pr
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7.0.1
|
||||
- name: Checkout ${{ matrix.ref }}
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
ref: ${{ matrix.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7.1.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run backend tests with coverage
|
||||
run: >-
|
||||
npm run test:backend --
|
||||
--coverage
|
||||
--shard=${{ matrix.shard }}/${{ strategy.job-total }}
|
||||
--reporter=default
|
||||
--reporter=blob
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload blob report
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
name: backend-blob-${{ matrix.shard }}
|
||||
path: .vitest-reports/
|
||||
include-hidden-files: true
|
||||
retention-days: 1
|
||||
|
||||
# Required check, so its PR name is pinned. A skipped job would satisfy it,
|
||||
# so this always runs and fails on any shard that didn't pass.
|
||||
test:
|
||||
needs: [changes, test-shard]
|
||||
if: ${{ !cancelled() }}
|
||||
name: test (${{ github.event_name == 'push' && 'main' || 'pr' }})
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check shard results
|
||||
env:
|
||||
CHANGES_RESULT: ${{ needs.changes.result }}
|
||||
BACKEND_CHANGED: ${{ needs.changes.outputs.backend }}
|
||||
SHARDS_RESULT: ${{ needs.test-shard.result }}
|
||||
run: |
|
||||
if [ "$CHANGES_RESULT" != "success" ]; then
|
||||
echo "changes job: $CHANGES_RESULT"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$BACKEND_CHANGED" != "true" ]; then
|
||||
echo "No backend changes."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$SHARDS_RESULT" != "success" ]; then
|
||||
echo "test shards: $SHARDS_RESULT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
coverage:
|
||||
needs: [changes, test-shard]
|
||||
if: ${{ !cancelled() && needs.test-shard.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
pattern: backend-blob-*
|
||||
path: .vitest-reports
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge shard coverage
|
||||
run: npx vitest run --config src/backend/vitest.config.ts --merge-reports --coverage
|
||||
run: npm run test:backend -- --coverage
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload coverage artifact
|
||||
uses: actions/[email protected]
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: backend-coverage-${{ github.event_name == 'push' && 'main' || 'pr' }}
|
||||
name: backend-coverage-${{ matrix.artifact }}
|
||||
path: src/backend/coverage/
|
||||
retention-days: ${{ github.event_name == 'push' && 30 || 14 }}
|
||||
retention-days: 14
|
||||
|
||||
# Latest main coverage from this repo's own runs; a fork can't publish
|
||||
# under this repository id.
|
||||
- name: Download main coverage
|
||||
id: base
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
id=$(gh api "repos/$REPO/actions/artifacts?name=backend-coverage-main&per_page=20" --jq '
|
||||
[.artifacts[]
|
||||
| select(.expired | not)
|
||||
| select(.workflow_run.head_branch == "main")
|
||||
| select(.workflow_run.head_repository_id == .workflow_run.repository_id)
|
||||
][0].id // empty')
|
||||
if [ -z "$id" ]; then
|
||||
echo "No main coverage artifact; reporting without a comparison."
|
||||
exit 0
|
||||
fi
|
||||
gh api "repos/$REPO/actions/artifacts/$id/zip" > coverage-base.zip
|
||||
unzip -q coverage-base.zip -d coverage-base
|
||||
echo "found=true" >> "$GITHUB_OUTPUT"
|
||||
report-coverage:
|
||||
needs: test
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download PR coverage
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: backend-coverage-pr
|
||||
path: src/backend/coverage
|
||||
|
||||
- name: Download base coverage
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: backend-coverage-base
|
||||
path: coverage-base
|
||||
|
||||
- name: Report coverage on PR
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: davelosert/[email protected]
|
||||
uses: davelosert/vitest-coverage-report-action@v2
|
||||
with:
|
||||
json-summary-path: src/backend/coverage/coverage-summary.json
|
||||
json-final-path: src/backend/coverage/coverage-final.json
|
||||
json-summary-compare-path: ${{ steps.base.outputs.found == 'true' && 'coverage-base/coverage-summary.json' || '' }}
|
||||
json-summary-compare-path: coverage-base/coverage-summary.json
|
||||
@@ -1,25 +0,0 @@
|
||||
name: Dependabot auto-merge
|
||||
|
||||
on: pull_request
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
automerge:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.actor == 'dependabot[bot]'
|
||||
steps:
|
||||
- name: Dependabot metadata
|
||||
id: metadata
|
||||
uses: dependabot/[email protected]
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Enable auto-merge for patch updates
|
||||
if: steps.metadata.outputs.update-type == 'version-update:semver-patch'
|
||||
run: gh pr merge --auto --squash "$PR_URL"
|
||||
env:
|
||||
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -45,32 +45,30 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to GitHub Package Container registry
|
||||
uses: docker/login-action@v4.8.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v4.8.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6.2.0
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
@@ -84,20 +82,18 @@ jobs:
|
||||
type=raw,value=main,enable=${{ github.event_name == 'push' || inputs.push_main }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v7.4.0
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
push: true
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Sync README to Docker Hub
|
||||
uses: peter-evans/dockerhub-description@v5.0.0
|
||||
uses: peter-evans/dockerhub-description@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
name: Docs Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/docs/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-docs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build docs
|
||||
run: npm run build
|
||||
working-directory: src/docs
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Maintain Release Merge PR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
update-release-pr:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup Git
|
||||
run: |
|
||||
git config user.name github-actions
|
||||
git config user.email [email protected]
|
||||
- name: Check for existing PR
|
||||
id: find-pr
|
||||
uses: juliangruber/find-pull-request-action@v1
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
branch: release
|
||||
- uses: actions/checkout@v4
|
||||
if: steps.find-pr.outputs.number == ''
|
||||
with:
|
||||
ref: release
|
||||
- name: Reset release branch
|
||||
if: steps.find-pr.outputs.number == ''
|
||||
run: |
|
||||
git fetch origin main:main
|
||||
git reset --hard main
|
||||
- name: Create/Update PR
|
||||
if: steps.find-pr.outputs.number == ''
|
||||
uses: peter-evans/create-pull-request@v6
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
commit-message: Update release branch
|
||||
title: Merge Release Auto-PR
|
||||
body: Merging this PR will invoke release actions
|
||||
branch: auto-update/release
|
||||
@@ -1,48 +0,0 @@
|
||||
name: Notify HeyPuter (dev)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
puter_ref:
|
||||
description: "puter branch, tag or SHA"
|
||||
type: string
|
||||
default: "main"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
authorize:
|
||||
if: github.repository == 'HeyPuter/puter'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Require maintain or admin
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ACTOR: ${{ github.triggering_actor }}
|
||||
run: |
|
||||
ROLE=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$ACTOR/permission" --jq .role_name)
|
||||
if [ "$ROLE" != "maintain" ] && [ "$ROLE" != "admin" ]; then
|
||||
echo "::error::$ACTOR has role '$ROLE'; maintain or admin required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
notify:
|
||||
needs: authorize
|
||||
runs-on: ubuntu-latest
|
||||
# Holds HEYPUTER_DISPATCH_TOKEN; limited to main so branch copies of this file can't read it.
|
||||
environment: heyputer-dispatch
|
||||
steps:
|
||||
- name: Trigger heyputer dev deploy
|
||||
env:
|
||||
PUTER_REF: ${{ inputs.puter_ref }}
|
||||
TOKEN: ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}
|
||||
run: |
|
||||
jq -nc --arg ref "$PUTER_REF" \
|
||||
'{event_type: "puter-dev-requested", client_payload: {puter_ref: $ref}}' |
|
||||
curl -fsS -X POST \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
https://api.github.com/repos/HeyPuter/heyputer/dispatches \
|
||||
-d @-
|
||||
echo "Requested \`$PUTER_REF\`" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -5,12 +5,9 @@ on:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
environment: heyputer-dispatch
|
||||
steps:
|
||||
- name: Trigger heyputer build
|
||||
run: |
|
||||
@@ -18,4 +15,4 @@ jobs:
|
||||
-H "Authorization: token ${{ secrets.HEYPUTER_DISPATCH_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
https://api.github.com/repos/HeyPuter/heyputer/dispatches \
|
||||
-d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
|
||||
-d '{"event_type":"puter-main-updated","client_payload":{"puter_ref":"main"}}'
|
||||
@@ -1,223 +0,0 @@
|
||||
name: Puter.js API Tests
|
||||
|
||||
# Always triggers so it can be a required PR check; the `changes` job below
|
||||
# decides whether puter.js-relevant paths actually changed (the suites live
|
||||
# in src/puter-js/tests, so test-only changes are covered by the same glob,
|
||||
# plus the pieces the runners depend on: the worker preamble, the in-memory
|
||||
# test env in testUtil, and the backend vitest config the API-tests config
|
||||
# extends), and every other job is skipped — not absent — when they didn't,
|
||||
# which still satisfies a required check. Backend and extension changes run
|
||||
# backend-tests.yaml instead.
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
puterjs: ${{ steps.filter.outputs.puterjs }}
|
||||
steps:
|
||||
- uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dorny/[email protected]
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
puterjs:
|
||||
- 'src/puter-js/**'
|
||||
- 'src/worker/**'
|
||||
- 'src/backend/testUtil.ts'
|
||||
- 'src/backend/vitest.config.ts'
|
||||
- 'tools/**'
|
||||
- 'package.json'
|
||||
- 'package-lock.json'
|
||||
- '.github/workflows/puterjs-tests.yaml'
|
||||
|
||||
# The JSDoc in src/puter-js/src is the source of truth for the SDK's public
|
||||
# types. The declarations shipped to npm are generated from it at build time
|
||||
# and never committed, so what needs guarding is the JSDoc itself: this
|
||||
# generates the declarations and type-checks the published surface without
|
||||
# skipLibCheck, which is how broken re-exports used to go unnoticed.
|
||||
#
|
||||
# Its own job rather than a step in `test`: that one builds bundles and
|
||||
# installs a browser, and this needs neither.
|
||||
types:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Check the puter.js JSDoc produces declarations that type-check
|
||||
run: npm run check:puterjs:types
|
||||
|
||||
test:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
# The runners exercise the built artifacts: the SDK bundle
|
||||
# (src/puter-js/dist) for node + browser, and the worker preamble
|
||||
# (src/worker/dist) for workerd.
|
||||
- name: Build puter.js SDK and worker preamble
|
||||
run: npm run build:workerLib
|
||||
|
||||
- name: Install Playwright chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run puter.js API tests (node, browser, workerd)
|
||||
run: npm run test:puterjs
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
# SDK coverage on both refs, reported as a PR comment — same shape as
|
||||
# backend-tests.yaml. This run uses the istanbul-instrumented bundle
|
||||
# (the `test` job above keeps exercising the production build).
|
||||
# `needs.changes` gates the steps below, not this job, on purpose: a
|
||||
# job-level `if` on a matrix job skips the whole job as one unexpanded
|
||||
# check ("coverage (${{ matrix.artifact }})", literally, since there's no
|
||||
# matrix context to fill it in) instead of two real per-artifact checks —
|
||||
# confusing to read, and unusable if this ever gets pinned as a required
|
||||
# check. Letting the matrix always expand keeps the check names stable.
|
||||
coverage:
|
||||
needs: changes
|
||||
# Overrides the default matrix naming, which would otherwise bake the
|
||||
# base/head branch name (matrix.ref) into the check name — making it a
|
||||
# different string on every PR and impossible to pin as a required check.
|
||||
name: coverage (${{ matrix.artifact }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# The base branch lives in this repo; for a fork PR, the head
|
||||
# repo's copy of it can be arbitrarily stale.
|
||||
- ref: ${{ github.base_ref }}
|
||||
repository: ${{ github.repository }}
|
||||
artifact: base
|
||||
- ref: ${{ github.head_ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
artifact: pr
|
||||
|
||||
steps:
|
||||
- name: Checkout ${{ matrix.ref }}
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ matrix.ref }}
|
||||
repository: ${{ matrix.repository }}
|
||||
|
||||
- name: Set up Node.js
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright chromium
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
# The base leg is best-effort — it only enriches the PR comment with
|
||||
# a comparison, and the base ref may predate the coverage script.
|
||||
- name: Run puter.js API tests with coverage
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
continue-on-error: ${{ matrix.artifact == 'base' }}
|
||||
run: npm run test:puterjs:coverage
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: needs.changes.outputs.puterjs == 'true' && !cancelled()
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
name: puterjs-coverage-${{ matrix.artifact }}
|
||||
path: src/puter-js/coverage/coverage-*.json
|
||||
retention-days: 14
|
||||
if-no-files-found: ignore
|
||||
|
||||
report-coverage:
|
||||
needs: [changes, coverage]
|
||||
if: always() && needs.changes.outputs.puterjs == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Missing when the PR leg failed; `coverage (pr)` already reports that.
|
||||
- name: Download PR coverage
|
||||
id: pr-coverage
|
||||
continue-on-error: true
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
name: puterjs-coverage-pr
|
||||
path: src/puter-js/coverage
|
||||
|
||||
# May not exist (base ref without the coverage script, or a failed
|
||||
# base run) — then report without the comparison column.
|
||||
- name: Download base coverage
|
||||
id: base-coverage
|
||||
if: steps.pr-coverage.outcome == 'success'
|
||||
continue-on-error: true
|
||||
uses: actions/[email protected]
|
||||
with:
|
||||
name: puterjs-coverage-base
|
||||
path: coverage-base
|
||||
|
||||
- name: Report coverage on PR
|
||||
if: steps.pr-coverage.outcome == 'success' && steps.base-coverage.outcome == 'success'
|
||||
uses: davelosert/[email protected]
|
||||
with:
|
||||
name: puter.js SDK
|
||||
json-summary-path: src/puter-js/coverage/coverage-summary.json
|
||||
json-final-path: src/puter-js/coverage/coverage-final.json
|
||||
json-summary-compare-path: coverage-base/coverage-summary.json
|
||||
|
||||
- name: Report coverage on PR (no base comparison)
|
||||
if: steps.pr-coverage.outcome == 'success' && steps.base-coverage.outcome != 'success'
|
||||
uses: davelosert/[email protected]
|
||||
with:
|
||||
name: puter.js SDK
|
||||
json-summary-path: src/puter-js/coverage/coverage-summary.json
|
||||
json-final-path: src/puter-js/coverage/coverage-final.json
|
||||
@@ -0,0 +1,18 @@
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
name: release-please
|
||||
|
||||
jobs:
|
||||
release-please:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: google-github-actions/release-please-action@v4
|
||||
with:
|
||||
release-type: node
|
||||
@@ -45,14 +45,14 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Log in to GitHub Package Container registry
|
||||
uses: docker/login-action@v4.8.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Create alias tag
|
||||
env:
|
||||
|
||||
@@ -66,13 +66,7 @@ AGENTS.md
|
||||
|
||||
|
||||
coverage/
|
||||
.vitest-reports/
|
||||
*.log
|
||||
undefined
|
||||
servers.json
|
||||
config.*.json
|
||||
|
||||
volatile/
|
||||
|
||||
# Local build override
|
||||
docker-compose.override.yml
|
||||
+1
-2
@@ -1,5 +1,4 @@
|
||||
{
|
||||
"tabWidth": 4,
|
||||
"singleQuote": true,
|
||||
"plugins": ["prettier-plugin-jsdoc"]
|
||||
"singleQuote": true
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for AI coding agents working in this repository. Read this before making changes. FOLLOW GUIDANCE AS CLOSELY AS POSSIBLE. If you think the guidance is wrong, raise an issue or flag a maintainer — don't just do what you think is right. This is the source of truth for how we write code, tests, and docs in this repo.
|
||||
|
||||
## Documentation Index
|
||||
|
||||
Use these as the source of truth before exploring further:
|
||||
|
||||
- [README.md](README.md) — project overview and quickstart.
|
||||
- [doc/architecture.md](doc/architecture.md) — backend layered stack (controllers → drivers → services → stores → clients), `PuterServer` wiring, `Context` (ALS), and extensions.
|
||||
- [doc/self-hosting.md](doc/self-hosting.md) — running Puter outside hosted infra.
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) — testing, security, AI-assisted code, PR conventions, Boy Scout Rule.
|
||||
- [SECURITY.md](SECURITY.md) — how to report vulnerabilities (do not file them publicly).
|
||||
- [BUG-BOUNTY.md](BUG-BOUNTY.md) — bounty program scope.
|
||||
- [TRADEMARK.md](TRADEMARK.md) — trademark usage.
|
||||
|
||||
---
|
||||
|
||||
## Backend
|
||||
|
||||
The backend is organized as a layered stack inspired by Controller–Service–Repository with dependency injection. Every layer only depends on the layers beneath it, and `PuterServer` ([src/backend/server.ts](src/backend/server.ts)) wires the whole thing together.
|
||||
|
||||
### Layers (top → bottom)
|
||||
|
||||
| Layer | Lives in | Responsibility |
|
||||
| --------------- | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Controllers** | [src/backend/controllers/](src/backend/controllers/) | Route handlers. Parse + validate input, apply per-route gates (auth, subdomain, rate limit, body parsers via `RouteOptions`), call services, format responses. |
|
||||
| **Drivers** | [src/backend/drivers/](src/backend/drivers/) | Optional. RPC-style handlers exposed over `/drivers/*`. Thin shells that validate RPC inputs and call into services/stores. |
|
||||
| **Services** | [src/backend/services/](src/backend/services/) | Business logic. Assume the caller is already authenticated/authorized — services do not run auth gates themselves. |
|
||||
| **Stores** | [src/backend/stores/](src/backend/stores/) | Persistence. Wraps clients with the domain shape services consume (rows, entities, KV namespaces). |
|
||||
| **Clients** | [src/backend/clients/](src/backend/clients/) | Adapters for external/internal services (sql, redis, s3, dynamo, email, event bus). Knows protocols, not domain concepts. |
|
||||
| **Config** | `config.*.json` → `IConfig` | Flat, typed config object every layer receives at construction. |
|
||||
|
||||
Each layer receives the layers beneath it through its constructor. Dependencies are explicit and traceable from `PuterServer`.
|
||||
|
||||
### Cross-layer rules
|
||||
|
||||
- **Don't reach across layers.** Controllers do not poke clients directly; services do not register routes. If you want to, the abstraction is wrong — fix the abstraction.
|
||||
- **Don't call sideways within a layer for code reuse.** If two services need the same logic, lift it into a util/helper. Services should not depend on other services for shared code.
|
||||
- **Prefer explicit arguments over `Context` (ALS).** Reach for [Context](src/backend/core/context.ts) only when the value is genuinely request-scoped and would otherwise thread through many layers. Today it's used sparingly — mostly for `actor` and `req`.
|
||||
|
||||
### Extensions
|
||||
|
||||
Extensions live in [extensions/](extensions/) and parallel the layered stack. They are for **non-crucial parts of the system** — things Puter still works without if removed. If a feature is load-bearing for clients, it belongs in core, not in an extension (see [whoami](extensions/whoami.ts) as the cautionary example).
|
||||
|
||||
The `extension` global ([src/backend/extensions.ts](src/backend/extensions.ts)) exposes:
|
||||
|
||||
- `extension.registerClient/Store/Service/Driver/Controller(name, Class)` for first-class additions.
|
||||
- `extension.on(event, handler)` and `extension.get/post/put/delete/patch/use(path, opts?, handler)` for the lightweight common case. `opts` is the same `RouteOptions` controllers use.
|
||||
- `extension.import('client' | 'store' | 'service' | 'controller' | 'driver')` returns a lazy proxy to instantiated objects. `extension.config` exposes live config.
|
||||
|
||||
### Language & file conventions
|
||||
|
||||
- **Modules:** We transpile and build as needed — write ES modules, not CommonJS.
|
||||
- **TypeScript preferred for new files.** Existing JS is fine; convert opportunistically when you're already touching a file.
|
||||
- **Reuse types before inventing them.** Search for an existing type first; extend it if close. Only define a new type when nothing fits.
|
||||
- **Make new types findable.** Co-locate them with the layer/module that owns them, export from the obvious entry point, and use a descriptive `PascalCase` name. Don't hide types in random files where future readers won't grep them.
|
||||
- **Naming:** `camelCase` for variables/functions, `PascalCase` for classes and for files containing a class (`AuthService.ts`, `KVStoreDriver.ts`).
|
||||
|
||||
### Comments
|
||||
|
||||
Keep comments light. Prefer self-documenting code — clear names, small functions, obvious flow. Add a comment **only** when:
|
||||
|
||||
- The _why_ is non-obvious (a hidden constraint, a workaround, a subtle invariant).
|
||||
- A non-trivial usage detail would otherwise trip the next reader.
|
||||
|
||||
Don't restate what the code already says. Don't write comments that reference the current task or PR — those rot.
|
||||
|
||||
### Tests
|
||||
|
||||
When adding new behavior (function, endpoint, driver method, branch of logic), add a test for it.
|
||||
|
||||
- Use Vitest for unit and integration tests. Test files go next to the code they test, named `*.test.ts` or `*.test.js`.
|
||||
- **Mock data, not methods.** Stub the inputs (fixtures, fake rows, sample payloads) rather than mocking the function under test or the layer beneath it. Over-mocking produces tests that pass while production breaks. If you must mock, mock at a real boundary (a client/external service), not within the same layer you're testing.
|
||||
- Prefer test server over mocking deps. We have existing test utilities for running full test server to boot deps based on config. (see [src/backend/testUtil.ts](src/backend/testUtil.ts)).
|
||||
- **Hit a real database / real client where reasonable.** Integration shapes catch the things unit tests with mocks miss.
|
||||
- **Regression tests for bug fixes.** A test that fails before your fix and passes after is the cheapest insurance against the bug coming back.
|
||||
- If something is genuinely hard to test (UI animation, third-party glue), skip it but say so in the PR.
|
||||
|
||||
### Security & privacy
|
||||
|
||||
Before opening a PR, scan the diff for:
|
||||
|
||||
- Logs, error messages, or responses leaking internal paths, secrets, tokens, env vars, or other users' data.
|
||||
- Debug routes, test credentials, commented-out auth checks.
|
||||
- Endpoints returning more than the caller actually needs.
|
||||
|
||||
When in doubt, return less. Auth-, permission-, or data-export-related changes deserve an explicit callout in the PR description.
|
||||
|
||||
### Working rules of thumb
|
||||
|
||||
- **Run it, don't just compile it.** "It type-checks" is not "it works." Exercise the code path end-to-end at least once.
|
||||
- **Read the neighbors before writing.** Match the shape of similar things already in the tree. If you genuinely think the existing pattern is wrong, raise it — don't quietly diverge.
|
||||
- **Boy Scout Rule, proportional to the change.** Fix the obvious typo, dead import, or missing typehint in files you're already touching. Don't ride a refactor along with a bug fix — that just makes review harder.
|
||||
- **Understand what you commit.** AI assistance is fine; shipping code you couldn't defend in review is not.
|
||||
@@ -1,236 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for AI coding agents working in this repository, and the source of truth for how we write code, tests, and docs here. Read it before making changes. FOLLOW GUIDANCE AS CLOSELY AS POSSIBLE. If you think a rule is wrong, raise an issue or flag a maintainer instead of doing what you think is right.
|
||||
|
||||
## Documentation Index
|
||||
|
||||
Use these as the source of truth before exploring further:
|
||||
|
||||
- [README.md](README.md) — project overview and quickstart.
|
||||
- [doc/architecture.md](doc/architecture.md) — backend layered stack (controllers → drivers → services → stores → clients), `PuterServer` wiring, `Context` (ALS), actors, and extensions.
|
||||
- [doc/contributing-apis.md](doc/contributing-apis.md) — adding and maintaining public APIs end to end (backend surface → puter.js → types → docs → tests). Follow it for any API work.
|
||||
- [doc/pagination.md](doc/pagination.md) — the one pagination convention for list APIs (limit/cursor/offset/includeTotal, envelope shape, cursor semantics).
|
||||
- [doc/alarms.md](doc/alarms.md) — raising alarms and picking a severity (what pages, what only gets recorded), plus the config that routes them.
|
||||
- [doc/self-hosting.md](doc/self-hosting.md) — running Puter outside hosted infra.
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) — contributor entry point and PR conventions.
|
||||
- [SECURITY.md](SECURITY.md) — how to report vulnerabilities (do not file them publicly).
|
||||
- [BUG-BOUNTY.md](BUG-BOUNTY.md) — bounty program scope.
|
||||
- [TRADEMARK.md](TRADEMARK.md) — trademark usage.
|
||||
|
||||
---
|
||||
|
||||
## Repo-wide conventions
|
||||
|
||||
These apply everywhere — backend, puter.js, and GUI.
|
||||
|
||||
### Language & files
|
||||
|
||||
- Write ES modules, not CommonJS — we transpile and build as needed.
|
||||
- TypeScript preferred for new files in the backend and extensions; convert existing JS there opportunistically when you're already touching a file. The GUI and puter.js are plain JavaScript — don't introduce TypeScript files in them.
|
||||
- In plain-JS files, typing with JSDoc (`@type`/`@param`/`@returns`, `@typedef` for shared shapes, using the TypeScript type system) is encouraged. API types must not be `unknown` or untyped `...args`: spell out the real parameter and return shapes, unless the value passes through transparently to an upstream layer that owns its type. puter.js has stricter rules ([below](#types)).
|
||||
- Make new types findable: descriptive `PascalCase` names, exported from the obvious entry point. A type used from many places goes in the owning module's `types.ts` (e.g. [src/backend/controllers/types.ts](src/backend/controllers/types.ts)); a type with a single consumer can stay next to it.
|
||||
- Naming: `camelCase` for variables and functions, `PascalCase` for classes and files containing a class (`AuthService.ts`). **Prefer `camelCase` for all new code**, including parameters and internal object properties. `snake_case` is reserved for external contracts that already use it: wire/JSON keys, stable API error codes, public option names that already ship as `snake_case`, and established `puter_*` namespaced fields. Don't rename those (it's a breaking change), and don't add new `snake_case` beside them; a camelCase local can carry a value into a wire key (`{ operation_id: operationId }`).
|
||||
|
||||
### Comments
|
||||
|
||||
Prefer self-documenting code. Comment only when the _why_ is non-obvious or a usage detail would trip the next reader. **A comment is a sentence or two**: one or two lines of `//`, a few of `/** ... */` JSDoc. State the constraint and stop. Don't restate the code, narrate history ("used to swallow the failure, which meant…"), argue the design, or explain parameters the signature already shows. Reasoning that needs paragraphs belongs in a design doc.
|
||||
|
||||
Don't reference the current task, PR, or version; those rot. **No ticket references** (`PUT-1234`, `// fix for FOO-99`) in code, comments, or test names: describe the why in domain terms. Section dividers use plain ASCII (`// -- Section --`), never box-drawing characters.
|
||||
|
||||
### Security & privacy
|
||||
|
||||
Before opening a PR, scan the diff for:
|
||||
|
||||
- Logs, error messages, or responses leaking internal paths, secrets, tokens, env vars, or other users' data.
|
||||
- Debug routes, test credentials, commented-out auth checks.
|
||||
- Endpoints returning more than the caller actually needs.
|
||||
|
||||
When in doubt, return less. Auth-, permission-, or data-export-related changes deserve an explicit callout in the PR description.
|
||||
|
||||
Commit messages and PR descriptions never credit anyone or mention a bug report. For security fixes, state what the change does mechanically ("scoped tokens do xyz now"), not the vulnerability or how it could be abused.
|
||||
|
||||
### Reuse and abstraction
|
||||
|
||||
- **Search before you build.** Before writing a type, helper, cache/TTL map, retry, lock, pagination, validator, serializer, error mapper, host/email normalizer, or provider adapter, grep `src/backend/util/`, the owning layer, and `extensions/` for one that exists, and extend it instead of adding a parallel copy. If you find two copies, consolidate them or flag it. Before opening the PR, check the diff for new helpers that duplicate existing ones.
|
||||
- **One path per operation.** Single-item variants call the batch version. Legacy and v2 endpoints share one service method. A second entry point (OIDC, WebDAV, admin, cron) calls the same service as the first instead of re-implementing it.
|
||||
- **No abstraction without a second caller.** No base class, interface, factory, registry, decorator, or option bag with one implementation. No wrappers that only forward, no interface methods nobody calls, no methods only tests call.
|
||||
- **Variants are data.** Providers or models that differ only in constants (base URL, prices, key names) are entries in a table read by one implementation, not one file each.
|
||||
|
||||
### Hot paths
|
||||
|
||||
- No per-row queries or awaits in loops: use the store's batch method (add one if missing) or [src/backend/util/concurrency.ts](src/backend/util/concurrency.ts).
|
||||
- Bound everything: queries get `LIMIT`/pagination, in-memory maps get eviction, outbound calls get a timeout and a size cap.
|
||||
- Build clients, catalogs, and config-derived lookups once, not per request.
|
||||
- Listeners on fleet-wide events (`fs.write.file`, route lifecycle) return before any I/O for requests they don't concern.
|
||||
|
||||
### Working rules of thumb
|
||||
|
||||
- **Run it, don't just compile it.** "It type-checks" is not "it works." Exercise the code path end-to-end at least once.
|
||||
- **Read the neighbors before writing.** Match the shape of similar things already in the tree. If you think the existing pattern is wrong, raise it — don't quietly diverge.
|
||||
- **Test new behavior.** Every new function, endpoint, driver method, or logic branch gets a test; every bug fix gets a regression test that fails before the fix. If something is genuinely hard to test, skip it but say so in the PR.
|
||||
- **Boy Scout Rule, proportional to the change.** In files you're already touching, fix the typo, the dead import, the missing test, the bit you had to read twice. Don't ride a refactor along with a bug fix.
|
||||
- **Understand what you commit.** AI assistance is fine; shipping code you couldn't have written, debugged, or defended in review is not.
|
||||
- **A limit change is not done until the docs change with it.** Every rate limit, concurrency cap, quota, and allowance in the code is published in [src/docs/src/rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) — an undisclosed limit is one developers discover as a service failure. If a PR moves any of these numbers, the same PR updates that page. The numbers live in `src/backend/controllers/fs/limits.ts` (filesystem), `src/backend/drivers/util/aiLimits.ts` (all AI drivers), per-driver `rateLimit`/`concurrent` configs, and `src/backend/data/subPolicies/` (free-tier allowances).
|
||||
|
||||
---
|
||||
|
||||
## Local development
|
||||
|
||||
To test a local Docker build, use the gitignored `docker-compose.override.yml` described in [Self-Hosting → Building from source](doc/self-hosting.md#building-from-source-instead-of-pulling). Keep local build settings there, not in `docker-compose.yml`.
|
||||
|
||||
When budget checks block unrelated local API tests, you may set `"unlimitedMetering": true` without further approval. Merge it into the gitignored `config.json` (`npm start`) or `puter/config/config.json` (Docker) and restart Puter. It covers every account on that instance, guests included, and usage is still recorded. Keep normal settings when testing budget or subscription enforcement, and leave shared defaults unchanged.
|
||||
|
||||
---
|
||||
|
||||
## Backend
|
||||
|
||||
Controllers → drivers (optional) → services → stores → clients, all configured from `IConfig`. Each layer depends only on the layers beneath it and receives them through its constructor; `PuterServer` ([src/backend/server.ts](src/backend/server.ts)) wires them. [doc/architecture.md](doc/architecture.md) is the full reference.
|
||||
|
||||
A public API can be a controller or a driver. Prefer a controller when you need fine-grained control over routes and gates; [doc/contributing-apis.md](doc/contributing-apis.md#controller-or-driver) has the decision guide.
|
||||
|
||||
Layer rules:
|
||||
|
||||
- **Don't reach across layers.** Controllers don't poke clients; services don't register routes; lower layers never import from controllers. If you want to, the abstraction is wrong; fix the abstraction.
|
||||
- **Gates run at the edge.** Controllers and drivers validate input and apply auth gates; services assume the caller is already authenticated and authorized.
|
||||
- **Don't call sideways within a layer for code reuse.** Two services needing the same logic means a util/helper, not a service-to-service dependency.
|
||||
- **A table's store owns every query against it.** Services, controllers, and extensions don't run raw SQL/redis against a table that has a store; add the store method so cache invalidation stays with the write.
|
||||
- **Drivers and controllers stay thin.** Domain rules a second entry point needs (signup, app CRUD, worker deploy, AI routing/billing) live in a service, not in the first driver or controller that needed them.
|
||||
- **Prefer explicit arguments over [`Context`](src/backend/core/context.ts) (ALS).** Use it only for request-scoped values that would otherwise thread through many layers; today that's mostly `actor` and `req`.
|
||||
|
||||
### Extensions
|
||||
|
||||
[extensions/](extensions/) is for **non-crucial parts of the system**: Puter must still work with the extension removed. If core needs to call it, it belongs in core ([whoami](extensions/whoami.ts) is the cautionary example). Inside an extension, follow the same layered structure, unless it only needs a few route handlers, in which case the `extension.get/post/...` helpers are enough on their own. The `extension` API is in [doc/architecture.md](doc/architecture.md#extension-api).
|
||||
|
||||
### Backend tests
|
||||
|
||||
- Vitest; test files sit next to the code they test (`*.test.ts` / `*.test.js`). Run with `npm run test:backend`.
|
||||
- **Mock data, not methods.** Stub inputs (fixtures, fake rows, payloads), not the function under test or the layer beneath it — over-mocking produces tests that pass while production breaks. If you must mock, mock at a real boundary (a client/external service).
|
||||
- **Prefer the test server over mocking deps.** `setupPuterTestEnv()` in [src/backend/testUtil.ts](src/backend/testUtil.ts) boots a fully in-memory backend; hit a real database/client shape where reasonable — integration shapes catch what mocked unit tests miss.
|
||||
- **Test code, not docs.** Never write a test that reads a file under `src/docs/` and asserts on its wording or numbers. Docs are kept in step by the PR (see the limits rule above) and checked in review; a test that greps a markdown page fails on every rewording and verifies nothing about behavior.
|
||||
|
||||
---
|
||||
|
||||
## puter.js (the SDK)
|
||||
|
||||
[src/puter-js/](src/puter-js/) is the public SDK. It ships live from `https://js.puter.com/v2/` with no version pinning, so every change reaches every existing app at once; see [Rule zero](doc/contributing-apis.md#rule-zero-dont-break-callers).
|
||||
|
||||
Layout: SDK modules in [src/puter-js/src/modules/](src/puter-js/src/modules/) (one file or directory per area — `FileSystem/`, `kv/`, `ai/`, …), shared plumbing in [src/puter-js/src/lib/](src/puter-js/src/lib/), generated (gitignored) type declarations in `src/puter-js/types/`, API tests in [src/puter-js/tests/api/](src/puter-js/tests/api/), UI e2e tests in [src/puter-js/tests/e2e/](src/puter-js/tests/e2e/), developer docs in [src/docs/](src/docs/).
|
||||
|
||||
Every SDK change carries all five of these, or the PR is incomplete. [doc/contributing-apis.md](doc/contributing-apis.md) has the steps for each.
|
||||
|
||||
1. **Backward compatibility**, unless a maintainer explicitly signs off on a break ([rules](doc/contributing-apis.md#maintaining-an-existing-api)).
|
||||
2. **Tests** in the cross-platform API suite, or the Playwright harness for desktop-rendered `puter.ui.*` ([steps](doc/contributing-apis.md#6-tests)).
|
||||
3. **Docs** in [src/docs/src/](src/docs/src/) that match the implementation exactly ([steps](doc/contributing-apis.md#5-docs), [writing rules](#developer-docs)).
|
||||
4. **Types**, per the rules below.
|
||||
5. **Error handling**: `{ message, code }` with stable `snake_case` codes, as in [modules/kv/](src/puter-js/src/modules/kv/) ([steps](doc/contributing-apis.md#3-puterjs)).
|
||||
|
||||
### Types
|
||||
|
||||
puter.js source is plain JavaScript typed via JSDoc, and **the JSDoc is the source of truth for the SDK's types**. `src/puter-js/types/` is `tsc --emitDeclarationOnly` output: the SDK build (`npm run build` in `src/puter-js`) generates it, the npm tarball ships it, and git ignores it. Never commit or hand-edit it. `npm run check:puterjs:types` generates it and type-checks the result without `skipLibCheck`; run it after any type change. The one hand-written declaration file is [src/puter-js/index.d.ts](src/puter-js/index.d.ts): it decides what is public and re-exports the generated names, so name a new type there if consumers should be able to import it.
|
||||
|
||||
- Public (exposed) methods carry JSDoc types for parameters and return value, with one `@overload` block per accepted call form; those overloads _are_ the published signature. Typing private helpers is at your discretion; annotate where it helps the next reader.
|
||||
- Tag members `@internal`, not `@private`, to strip them from the generated declarations.
|
||||
- Declarations must match runtime behavior exactly. A wrong type is worse than a missing one.
|
||||
|
||||
Declare a shape where it belongs and reference it elsewhere with an `import(...)` type. A shape more than one file needs lives in the module's `types.js`; one shared across modules lives in [src/puter-js/src/lib/types.js](src/puter-js/src/lib/types.js); one with a single consumer can stay next to it:
|
||||
|
||||
```js
|
||||
/** @typedef {import('./types.js').KVOptConfig} KVOptConfig */
|
||||
```
|
||||
|
||||
Use `@typedef {Object}` + `@property` for any shape whose fields need documenting; it's the only JSDoc form that carries a per-field doc comment into the generated declarations. Use the inline form for small internal shapes with nothing to say per field, and prefer `unknown` over `*`:
|
||||
|
||||
```js
|
||||
/** @typedef {{ key: string; value: unknown }} KVEntry */
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Developer docs
|
||||
|
||||
[src/docs/src/](src/docs/src/) is the published puter.js documentation: one reference page per method (`<Area>/<method>.md`, [steps](doc/contributing-apis.md#5-docs)) and recipes in [src/docs/src/recipes/](src/docs/src/recipes/).
|
||||
|
||||
### Page structure
|
||||
|
||||
The reference has two kinds of page:
|
||||
|
||||
- **API overview pages** (`<Area>.md`, such as [KV.md](src/docs/src/KV.md)) give a quick top-level picture of what the API is for and what it can do, with short examples and a list of its functions that links every function page in the area. Leave out the details. The first sentence of the intro says what the API is about. This is a judgment call too: when the API depends on a Puter concept the reader may not know, such as Teams, the first sentence can name it and the sentences after it explain it.
|
||||
- **Function pages** (`<Area>/<method>.md`) hold the details: parameters, defaults, return values, errors, limits, and edge cases.
|
||||
|
||||
An overview page answers "what is this API and is it what I need?", and a function page answers "how exactly does this call behave?". A change to one function usually only touches that function's page, so a new `kv.list()` option goes in `KV/list.md` without a matching update to `KV.md`. Don't copy a function's options, limits, or edge cases onto the overview page for completeness. Where a detail goes is a judgment call, not a hard rule: update the overview when a change affects how someone would describe the API as a whole.
|
||||
|
||||
When you add a page, add it to [src/docs/src/sidebar.js](src/docs/src/sidebar.js) under its area, or it won't show up in the navigation.
|
||||
|
||||
**Sidebar order.** Order the API areas in the sidebar by how often developers use them, most common first. Overview stays first and Objects stays last. Inside an area, order the pages by use too, and keep pages that make up one feature next to each other, such as `share()`, `unshare()`, `listShared()`, and `listSharedByMe()`. A group goes where its most-used page would go. A helper that only supports another function, such as `listModels()` for `chat()`, goes by its own use. Put create, list, get, update, and delete in that order, and constants such as `MAX_KEY_SIZE` last. The function list on the area's overview page uses the sidebar's order. When you add a runnable example, add its HTML to `src/docs/src/playground/examples/<slug>.html`, register the slug in [src/docs/src/examples.js](src/docs/src/examples.js), and link the code block to it by putting the slug after the language with a semicolon (` ```html;kv-set `).
|
||||
|
||||
### Function page layout
|
||||
|
||||
A function page opens with one sentence that says what the function does, in its own paragraph, such as "Given a prompt, generate an image using AI." Put any more detail in the paragraphs after it.
|
||||
|
||||
After the intro, a function page has these `##` sections, in this order:
|
||||
|
||||
1. Syntax
|
||||
2. Parameters
|
||||
3. Return value
|
||||
4. Errors, if the function has any
|
||||
5. Sections for specific topics, if the page needs them, with `## Model-specific features` last
|
||||
6. Examples
|
||||
|
||||
[chat.md](src/docs/src/AI/chat.md) follows this layout.
|
||||
|
||||
- **Options as a table.** Document an options object as a table with Option, Type, and Description columns. Give each option its own row, even when two options go together, such as `width` and `height`. Order the rows by how often developers use them, most common first, without group rows. In chat.md that's the core options, then reasoning, tools, response handling, request control, and output.
|
||||
- **Common options in the table, model-specific behavior in its own section.** A page whose function works with many models or providers, such as [chat.md](src/docs/src/AI/chat.md) and [txt2img.md](src/docs/src/AI/txt2img.md), splits its options this way:
|
||||
- **The options table** under Parameters lists the options that work with every model, and describes each one the way it works for every model.
|
||||
- **`## Model-specific features`**, the last section before Examples, covers everything else. It has a `###` for each model family or provider, named after it, such as "OpenAI models" or "Cloudflare models". Under it go the options only some models accept, and the common options a model handles differently, such as a narrower range, other allowed values, or an option it ignores.
|
||||
- **Write there only what's different.** A `###` lists an option only when that model or provider adds it or handles it differently. An option that works the same everywhere stays out, even if the provider supports it, and a model or provider with nothing different gets no `###`.
|
||||
- **Model-specific features is the only place for model-specific details.** Every limit, format, allowed value, default, price, or quirk that applies to some models goes there, in that model's or provider's `###`, and nowhere else. The options table and topic sections (such as Image inputs or Video input) describe what's true for every model and link to Model-specific features for the rest. Don't add a table or sentence there that compares providers or repeats a model's limits. Each copy is another source of truth to keep in sync, and copies drift when a model changes.
|
||||
- **Link the two.** Directly under the options table, put `Some models accept additional options or handle these options differently. See [Model-specific features](#model-specific-features).` A row whose option behaves differently on some models can also end with `See [Model-specific features](#model-specific-features).`
|
||||
- **A long topic gets its own `##` section.** When explaining an option or feature takes more than a sentence or two, such as function calling or output dimensions, give it a section between Errors and Examples and link to it from the options table. This keeps the reference sections short.
|
||||
- **Usage patterns are examples.** A way of using the API, such as stopping a stream, goes in a runnable example that the relevant section links to ("See the [stop a stream example](/playground/ai-chat-stop-stream/)."), not in a section of its own.
|
||||
- **Each example has a `###` heading** under `## Examples`, so every example shows in the table of contents.
|
||||
|
||||
### Writing docs
|
||||
|
||||
These apply to reference pages and recipes alike.
|
||||
|
||||
- **Document the developer-facing side only.** The reader is building on top of puter.js, so cover what affects their code: signatures, defaults, return shapes, errors, limits, and behavior they can observe. Leave out internal workings (which service handles the call, storage layout, caching, retries) unless they change what the developer has to do. [rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) is the exception: it publishes every limit, including timeouts and caps that come from how the backend works.
|
||||
- **Document Puter as it runs on puter.com.** The docs are for developers building on production Puter. Leave out behavior that only happens on a self-hosted or differently configured server, such as a provider that isn't set up there.
|
||||
- **Describe the current state.** Docs aren't a changelog. Write how the API behaves now, without "now", "no longer", "existing code is unaffected", or other wording that compares it to an older version.
|
||||
- **Verify before you document.** Run the calls you describe against a running Puter instead of inferring behavior from the source.
|
||||
- **Write plainly.** Use the literal words a developer would say out loud, with contractions and everyday words. No idioms or figurative verbs ("walk away from something shared with you" → "remove your own access", "nothing to page through" → "without pagination"). No formal register ("present only while more pages remain" → "includes a `cursor` if there are more pages to load"). Prefer dev terms such as atomic, concurrent, and insertion order.
|
||||
- **State the rule, then its consequence, in short sentences.** No metaphor in place of a rule. Name a concrete action ("you can create any files and folders inside it") over an abstract permission ("yours to organize freely"). Cut a sentence that only repeats the next one.
|
||||
- **Describe behavior directly.** No quoted inner-voice phrases. Write "`isNew` is `true` when the person did not have access before", not "this tells 'I just gave this to someone' from 'they already had it'".
|
||||
- **Keep references easy to follow.** Name what a list or code block belongs to in the sentence directly above it. Name a thing once and keep that name. Introduce an actor as "a user" before giving them a role: "A user can create a team. The user who creates the team becomes its owner."
|
||||
- **Sentence mechanics.** Never start a sentence with inline code: write "To store data, use the [`set()`](/KV/set/) method." Link every inline mention of an API to its page, every time. No em dashes, and no colons or semicolons mid-sentence; a colon that introduces a code block is fine.
|
||||
- **No `##` or `###` headings inside tabs.** The table of contents lists every `##` and `###` heading on the page, including the ones in a hidden tab (an `example-content` block switched by the `example-group` pills), so its links lead to content the reader can't see. Put the heading above the tabs, or use `####` inside a tab, which the table of contents skips.
|
||||
- **Show the correct way.** No "don't" and no wrong-way-first examples, in recipes and playground examples alike. Mention a wrong approach only when developers already hit it in practice, and state how it fails as a fact: "`add()` reads a plain object as a path map, so the call rejects with …".
|
||||
|
||||
### Recipes
|
||||
|
||||
Recipes are tested, copy-pasteable patterns for a task, one layer above the per-method reference, often combining several APIs. Agents find them through the `## Recipes` section of `llms.txt`.
|
||||
|
||||
- **Files.** One `.md` per recipe in [src/docs/src/recipes/](src/docs/src/recipes/), with `title`, `description`, `tags`, and optional `order` front matter; they're picked up automatically. Tags must exist in [src/docs/src/recipe-tags.js](src/docs/src/recipe-tags.js) or the build fails. Quote a `description` that contains a colon followed by a space. Every top-level `.md` in that directory is built as a recipe, so keep other markdown out of it.
|
||||
- **Title.** Short, Title Case, starts with a verb, and leaves out "to Your App" ("Add Team Support", "Deploy Workers Programmatically"). The slug is the title in kebab case, with no API-name prefix. In `llms.txt` an agent sees only the title and description before deciding to open a recipe, so word them to match the task the reader is trying to do.
|
||||
- **Description.** Written for someone who has never heard of Puter. Start with "Learn how to", put the action first, and say what the reader's app gains: "Learn how to add file sharing to your app with Puter.js, so users can share files in their Puter account with other people." No API names, no mechanism, no list of example cases, and no "not just …" contrast.
|
||||
- **Intro.** Explain the situation and why you'd do this, tied to something the reader already knows ("`puter.kv` is your database in Puter.js"). No "Use this when" bullet list.
|
||||
- **Structure.** Get to the first working call as early as possible, then add validation, rate limits, and sign-in after it. Explain a Puter concept the first time it appears, such as what `me.puter` is or that `puter.workers.exec()` opens the Puter sign-in window. Split recipes by what the reader needs, not by technique, and give a family of recipes a shared tag.
|
||||
- **Reading order.** A recipe reads top to bottom. Each code block uses only functions and variables from blocks above it, so the reader never has to search further down the page to understand it. Start with a small version that works, and have each section add one case. When a section changes a function from earlier, show the whole new version of that function there. If the code is built up over several sections, end with a `## Full Code` block that has all of it.
|
||||
- **Link to the reference for full option lists.** Mention only the options the recipe uses, then link to the function page for the rest ("All the sort options are listed on [`puter.fs.readdir()`](/FS/readdir/)."). A copied list of options, rules, or scopes goes stale when the reference changes.
|
||||
- **Limits.** Mention a limit only when it's the reason for a pattern the recipe uses, such as one wildcard subscription instead of one per key, and link to [rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) or the function page for the number. Leave out limits that code following the recipe never reaches.
|
||||
- **Code.** Platform-neutral: just the code, with top-level `await`. No `<script>` or `<html>` wrapper, no IIFE.
|
||||
- **Tradeoffs.** Justify a choice with one scenario the reader can picture right away, in about two short paragraphs, or fold each point into the section where it matters. The description states the benefit only.
|
||||
- **Notes.** Leave out `## Notes` by default. Use it only to point to an architecturally different approach. A limit the recipe needs goes in the section it applies to, and other facts belong on the linked reference page.
|
||||
|
||||
---
|
||||
|
||||
## GUI
|
||||
|
||||
[src/gui/](src/gui/) is the Puter desktop: deliberately plain JavaScript + jQuery with HTML-string templates. Don't introduce a UI framework or a new rendering pattern.
|
||||
|
||||
**Conformity over novelty**: match the existing design and code structure even where you'd choose differently. A visually or structurally divergent addition is a defect even when it works.
|
||||
|
||||
- **Reuse existing UI primitives before writing new ones.** Windows and dialogs are `UIWindow*` functions in [src/gui/src/UI/](src/gui/src/UI/); generic pieces already exist (`UIAlert`, `UIPrompt`, `UIContextMenu`, `UINotification`, `UIPopover`, widgets in [UI/Components/](src/gui/src/UI/Components/)). A new window should read like its neighbors: an async function taking an options object, composing an HTML string, wiring behavior with jQuery, delegating to `UIWindow(...)`.
|
||||
- **Match the visual language.** Use existing CSS classes (`button`, `button-primary`, window chrome, form styles) and copy the layout patterns of neighboring windows; new styles go in [src/gui/src/css/](src/gui/src/css/) following existing conventions. Verify anything positional (menus, overlays, z-index) on both desktop and mobile viewports.
|
||||
- **i18n every user-facing string.** No hardcoded UI text — use `i18n('key')` and add the key to [src/gui/src/i18n/translations/en.js](src/gui/src/i18n/translations/en.js).
|
||||
- **Shared logic goes in helpers/services.** Reusable non-UI logic belongs in [src/gui/src/helpers/](src/gui/src/helpers/) or [src/gui/src/services/](src/gui/src/services/), not copy-pasted between windows.
|
||||
- **Tests.** Vitest is wired for the GUI (`src/**/*.test.js`; see [appOrder.test.js](src/gui/src/UI/Dashboard/appOrder.test.js) for the shape). Extract pure logic into functions and test those. Desktop behavior driven through puter.js (`puter.ui.*`) is covered by the Playwright harness in [src/puter-js/tests/e2e/](src/puter-js/tests/e2e/) — add a spec there when you change how the desktop renders SDK-driven UI.
|
||||
+17
-79
@@ -6,100 +6,38 @@ We at **Puter** are committed to maintaining a secure experience for our users a
|
||||
|
||||
The following are in scope for this program:
|
||||
|
||||
- **The Puter open-source project** (available at [github.com/HeyPuter/puter](https://github.com/HeyPuter/puter))
|
||||
- **`puter.com`**
|
||||
- **`api.puter.com`**
|
||||
* **The Puter open-source project** (available at [github.com/HeyPuter/puter](https://github.com/HeyPuter/puter))
|
||||
* **`puter.com`**
|
||||
* **`api.puter.com`**
|
||||
|
||||
Out-of-scope:
|
||||
|
||||
- Third-party services, applications, or libraries not maintained by Puter.
|
||||
- Social engineering attacks (e.g., phishing against staff).
|
||||
- Denial of Service (DoS), spam, or volumetric attacks.
|
||||
- Physical security issues.
|
||||
|
||||
## Eligibility
|
||||
|
||||
We pay bounties for issues that threaten user security and privacy, or the security and health of the platform. We appreciate every report, but hardening suggestions and correctness bugs are not eligible, even when we fix them. Please open a GitHub issue for those.
|
||||
|
||||
A report must show how an attacker can directly perform the action against another account or the system. It is not eligible if it:
|
||||
|
||||
- Starts from access the attacker should not already have: a stolen session or token, the victim's password, a root or full-access token, or script execution on `puter.com`.
|
||||
- Needs the victim to do more than ordinary use (opening a link or an app), such as publishing their own secrets, sending a verification link to an address they don't control, or approving a prompt that accurately states what it grants.
|
||||
- Gives the attacker nothing they couldn't already get through a supported API.
|
||||
- Only reproduces in a non-default or development setup (SQLite, a `custom` OIDC provider, an unset outbound proxy, cookies or config planted by hand). Findings in the open-source code must affect `puter.com` or a default self-hosted install.
|
||||
|
||||
**Shared state.** Sharing files, KV entries, or cross-app KV access means the sharer trusts the sharee. Exploits that depend on that trust, such as a sharee misusing access the sharer granted, may be treated as hardening or lower severity.
|
||||
|
||||
Hardening we ship in a public PR does not make similar code paths vulnerabilities. They are judged by these same rules.
|
||||
|
||||
## Known Non-Issues (Please Check Before Submitting)
|
||||
|
||||
The following have already been reviewed and determined **not to be vulnerabilities**. Reports that only re-describe one of these are **not eligible for a reward** and will be closed as non-issues — even if they include new code references. Please review this list before submitting:
|
||||
|
||||
- **XSS / CORS / token issues scoped to `api.puter.com`.** The primary user session cookie lives on `puter.com`, not on the API origin, so reflected/stored XSS, CORS, or token handling on `api.puter.com` is out of scope. This explicitly includes **missing `Content-Security-Policy: sandbox` on any endpoint that serves a file inline** — `/token-read`, `/file`, `/fs/read`, WebDAV, and any MIME type the `applyInlineContentSecurity` allow-list does not cover (`text/xsl`, `image/svg+xml`, `application/xhtml+xml`, …). Script execution on the API origin has nothing to reach: there is no session cookie, no `localStorage`, and no ambient credential there. WebDAV additionally has to return an accurate MIME type for real WebDAV clients. The one exception we _do_ evaluate on its merits is anything that abuses the app-scoped `puter_token_v2` companion cookie set on the API host — if your report does not name a specific credential that is actually reachable from the API origin, it is this non-issue.
|
||||
- **SSRF via `secureFetch`.** Production routes outbound requests through an isolated proxy that has no access to internal/SSRF-sensitive resources.
|
||||
- **Attacks that depend on guessing an `appInstanceID` or app UID.** These are random 128-bit secret values and are not considered guessable.
|
||||
- **Apps invoking drivers, creating workers, or using KV.** Applications are intended to do this; worker permissions are scoped to the owning app. This is by design.
|
||||
- **App metadata or app user-count "leaks".** This information is currently public by design. This covers **private apps** too: private apps are an unreleased, admin-only beta for paid listings in the Puter App Center. Their `index_url` and other metadata are not secret, the flag is not user-settable, and no production app relies on it. Reports that a private app's metadata is reachable through a sibling route (`/get-launch-apps`, `index_url` drift on the hosting gate, etc.) are normalization work, not vulnerabilities.
|
||||
- **Worker hot-reload driven by a share recipient.** Sharing a worker's source file with write access is how you let a collaborator change that worker's code and have it deploy live. The redeploy running as the file owner, with the owner's worker credentials, is the intended behavior of that grant — the owner chose to hand out write access to executable code. If you do not want that, do not share the source file with write access.
|
||||
- **Re-claiming a site or worker name its owner deleted.** A hosted site or worker name that its owner deletes is treated as abandoned by that owner. If one of their apps still points at it, whoever claims the name next serves that app's `index_url` and receives what the app is launched with, and a new app on the same origin may share the old one's identity. Guarding app identities against being recycled this way is ongoing hardening, not a security boundary, so new ways to re-claim an abandoned name, or to inherit an app's tokens or data through one, are not eligible.
|
||||
- **Share-mode downgrades that do not cascade.** Demoting a delegate does not retroactively revoke the grants that delegate already issued downstream. Shares are independent grants once issued, not a live-evaluated chain. Revoking the downstream grant is a separate, supported action.
|
||||
- **Metering, credit-hold, and quota inaccuracies.** Credit holds and quota checks are up-front estimates, and actual usage is billed afterwards. Overspend that is bounded and still billed is a correctness bug, including concurrent requests that exceed a hold or quota by some margin; the per-account concurrency and rate-limit gates are the control we rely on. Report these only if you can use a service without it being billed, or overspend without bound.
|
||||
- **Existence-only oracles.** A response that tells "exists but forbidden" apart from "not found" for a path or ID the caller already supplied, without revealing anything else about it, is hardening.
|
||||
- **Session and credential lifecycle around 2FA enrollment.** Enrolling in 2FA is not a credential-revocation event: existing authenticated sessions stay valid, and outstanding password-reset links stay valid. Likewise, account-management actions behind the `/user-protected/*` gate are password-gated by design, not TOTP-gated. A recovery code that stops being accepted fails _closed_ — it denies access rather than granting it, so it is a bug report, not a vulnerability.
|
||||
- **Group permissions** (`/auth/revoke-user-group` and siblings). The group-permission surface is unfinished and not exposed anywhere in the product. Findings there are not reachable by any user.
|
||||
- **Missing Subresource Integrity (SRI) on external scripts/styles.** Accepted as hardening, not as a vulnerability, absent a demonstrated compromise of the CDN in question.
|
||||
- **General "token in a URL" / token-lifetime / token-storage designs** — signed directory URLs exposing children, a write signature implying read, app tokens outliving a web session, or the session token being held in `localStorage` on `puter.com`. These are current intended behaviors. Shorter-lived tokens with a refresh flow are on the roadmap; their absence is not a vulnerability on its own. Show a reproducible attack that does not already presuppose script execution on `puter.com`.
|
||||
- **Missing PKCE, unverified `id_token` signatures, or other OIDC hardening.** Provider tokens are obtained through a server-to-server authorization-code exchange with the provider's token endpoint over TLS, so the resulting `id_token` / userinfo claims are trusted from that channel rather than from local JWKS signature verification — the callback never accepts a caller-supplied `id_token`. Adding local signature / `aud` / `iss` / `exp` checks is welcome defense-in-depth (please open a GitHub issue/PR), but their absence is not an account-takeover vector on its own.
|
||||
- **JWT "algorithm confusion" / unpinned `algorithms` in `jwt.verify`.** Puter's session tokens are HMAC-signed (HS256) with a server-side secret, and there is no asymmetric public key anywhere in the verification path, so `alg`-substitution attacks do not apply. Explicitly pinning `algorithms` is a fine hardening PR, but it is not a vulnerability.
|
||||
- **Static-source "SQL injection" in internal pagination.** Findings such as `LIMIT ${limit}` in list/notification queries: the limit is numerically coerced and clamped before it reaches the query, so it is not reachable with attacker-controlled string input. Hardening PRs to the internal stores are welcome, but these are not exploitable as reported.
|
||||
- **Deprecated `saveTo*` GUI app messages.** The legacy `saveToDesktop` / `saveToDocuments` / etc. app-IPC handlers can create — never overwrite — new files in standard user folders. The behavior is non-destructive, path-traversal-safe, and deprecated (slated for removal); it is not treated as a sandbox escape.
|
||||
- **Best-practice suggestions** such as login/registration username enumeration (kept intentionally for UX) or unauthenticated unsubscribe links (industry norm).
|
||||
- **Rate-limiting suggestions for TURN credential issuance** (intentional; not billed per tunnel).
|
||||
|
||||
If you believe you have a **genuinely new** exploit chain that defeats one of these rationales (for example, demonstrating a sensitive credential that really is reachable on `api.puter.com`), say so explicitly and show why the reasoning above does not apply.
|
||||
* Third-party services, applications, or libraries not maintained by Puter.
|
||||
* Social engineering attacks (e.g., phishing against staff).
|
||||
* Denial of Service (DoS), spam, or volumetric attacks.
|
||||
* Physical security issues.
|
||||
|
||||
## Rules of Engagement
|
||||
|
||||
To participate, you must:
|
||||
|
||||
1. **Report responsibly**: Provide detailed steps to reproduce the issue, including proof-of-concept code, screenshots, or a screen recording (see _Proof of Reproduction_ below).
|
||||
1. **Report responsibly**: Provide detailed steps to reproduce the issue, including proof-of-concept code or screenshots where applicable.
|
||||
2. **Do no harm**: Do not exfiltrate, modify, or delete data. Only access your own account or test data.
|
||||
3. **Respect availability**: Do not perform denial-of-service attacks or automated scans that degrade service.
|
||||
4. **Follow disclosure policy**: Do not publicly disclose vulnerabilities until we have confirmed and patched the issue.
|
||||
5. **Act in good faith**: Make every effort to avoid privacy violations, destruction of data, and interruption or degradation of services.
|
||||
6. **Check Eligibility and Known Non-Issues**: Reports that fall outside "Eligibility" or match an item in "Known Non-Issues" above are not eligible and will be closed.
|
||||
|
||||
Reports that do not meet these guidelines may not be eligible for a reward.
|
||||
|
||||
## Proof of Reproduction
|
||||
|
||||
Reports **MUST** demonstrate a **working, reproducible exploit with real impact** — not a theoretical or static-source-review finding. Please include:
|
||||
|
||||
- Exact steps to reproduce, the relevant request/response or code path, and the commit or version you tested.
|
||||
- The **observed** result versus the **expected** result.
|
||||
- For client-side, UI, or authentication-flow bugs: a short screen recording (≤ 2 minutes) showing the exploit working end-to-end on a real Puter instance.
|
||||
- For server-side bugs: a runnable proof-of-concept and screenshot or video showing the attack process. A unit test or in-process harness alone does not count; show it against a running instance.
|
||||
|
||||
Reports based solely on reading the source ("source review only, not tested") or unverified AI/LLM-generated reports are the **lowest triage priority and not eligible for bounty**.
|
||||
If you used an AI tool to help find an issue, you must personally verify that it actually reproduces before submitting. Reports sent by automated agents are not eligible.
|
||||
|
||||
Please submit **one issue per report**.
|
||||
Bundled "audit packs" of many speculative findings will not be eligible for bounty; send each confirmed issue separately.
|
||||
Additionally please **DO NOT** submit too many issues at once; these will be treated as a bundled report, and will not be eligible for bounty.
|
||||
|
||||
## Reporting Process
|
||||
|
||||
To report a vulnerability, email us at: **[[email protected]](mailto:[email protected])**.
|
||||
Include:
|
||||
|
||||
- A description of the vulnerability
|
||||
- Steps to reproduce
|
||||
- Potential impact
|
||||
- Suggested remediation (if available)
|
||||
- Paypal id where we can send the bounty (if eligible)
|
||||
|
||||
Send the full report in your first email. We don't reply to messages that only ask whether we run a bounty or where to report.
|
||||
* A description of the vulnerability
|
||||
* Steps to reproduce
|
||||
* Potential impact
|
||||
* Suggested remediation (if available)
|
||||
|
||||
We aim to acknowledge receipt within **72 hours** and provide a resolution timeline.
|
||||
|
||||
@@ -107,13 +45,13 @@ We aim to acknowledge receipt within **72 hours** and provide a resolution timel
|
||||
|
||||
We offer monetary rewards based on the severity of the vulnerability, as determined by our internal assessment (using CVSS as a guide).
|
||||
|
||||
- **Critical: \$1,000 – \$2,000**
|
||||
- **High: \$500 – \$1,000**
|
||||
- **Medium: \$200 – \$500**
|
||||
- **Low: \$50 – \$100**
|
||||
* **Critical: \$1,000 – \$2,000**
|
||||
* **High: \$500 – \$1,000**
|
||||
* **Medium: \$200 – \$500**
|
||||
* **Low: \$50 – \$100**
|
||||
|
||||
Non-security issues, suggestions, and best practices feedback are always welcome, but may not qualify for a reward.
|
||||
If multiple researchers report the same issue, the bounty will be awarded to the first eligible report we receive. Issues we already know about, from internal review, an open ticket, or a fix already in a public commit or PR, count as duplicates.
|
||||
If multiple researchers report the same issue, the bounty will be awarded to the first eligible report we receive.
|
||||
|
||||
## Payments Disclaimer
|
||||
|
||||
|
||||
+32
-11
@@ -1,21 +1,42 @@
|
||||
# Contributing to Puter
|
||||
# Contributing to Puter Backend
|
||||
|
||||
Thanks for contributing. These rules aren't strictly enforced, but following them makes every PR easier. If anything's unclear, ping a core maintainer or open the PR and ask.
|
||||
Thanks for contributing. These rules aren't strictly enforced — but following them makes every PR easier. If anything's unclear, ping a core maintainer or open the PR and ask.
|
||||
|
||||
- New to the backend? Start with [doc/architecture.md](doc/architecture.md).
|
||||
- Code, test, and docs conventions for the whole repo live in [AGENTS.md](AGENTS.md). It's written for AI agents; the conventions apply to human contributors too.
|
||||
- Adding or changing a public API (an endpoint, driver method, or puter.js method)? Follow [doc/contributing-apis.md](doc/contributing-apis.md): backward compatibility, [developer docs](src/docs/), types, and tests all move in the same PR.
|
||||
- Found a vulnerability? Report it privately per [SECURITY.md](SECURITY.md).
|
||||
New to the backend? Start with [doc/architecture.md](doc/architecture.md).
|
||||
|
||||
---
|
||||
|
||||
## Before you open a PR
|
||||
## 1. Test it. Run it.
|
||||
|
||||
Check your diff against these sections of AGENTS.md:
|
||||
Run the affected code path end-to-end before opening a PR. "It builds" is not "it works."
|
||||
|
||||
- [Working rules of thumb](AGENTS.md#working-rules-of-thumb): run it end to end, test new behavior and bug fixes, understand every line you commit (AI assistance is fine), leave touched files a bit better, and publish limit changes.
|
||||
- [Reuse and abstraction](AGENTS.md#reuse-and-abstraction) and [hot paths](AGENTS.md#hot-paths).
|
||||
- [Security & privacy](AGENTS.md#security--privacy): expose nothing the caller doesn't need.
|
||||
Add tests for new behavior, endpoints, or bug fixes. If something's genuinely hard to test, say so in the PR.
|
||||
|
||||
## 2. Follow existing patterns
|
||||
|
||||
Match the shape of similar code already in the repo. [doc/architecture.md](doc/architecture.md) is the source of truth for layers, wiring, and naming. If you think a pattern is wrong, raise it — don't quietly diverge.
|
||||
|
||||
## 3. Don't expose system or user information
|
||||
|
||||
Scan your diff for stray logs, debug routes, internal paths, secrets, tokens, or user data in errors/responses. When in doubt, return less. Flag any auth, permission, or data-export changes in the PR description.
|
||||
|
||||
For private security reports, see [SECURITY.md](SECURITY.md).
|
||||
|
||||
## 4. AI-assisted code is fine — understood code is required
|
||||
|
||||
Don't commit code you couldn't have written, debugged, or defended yourself. Read the diff, run it, and be ready to explain it in review.
|
||||
|
||||
## 5. Update docs for API changes
|
||||
|
||||
If you change puter-js APIs (drivers or endpoints used by puter-js), update [developer docs](src/docs/). puter-js itself may also need updating.
|
||||
|
||||
## 6. Boy Scout Rule — leave it 1% better
|
||||
|
||||

|
||||
|
||||
Fix the typo, the dead import, the missing test, the bit you had to read twice. Keep cleanup proportional to the change — no refactors riding along on bug fixes.
|
||||
|
||||
---
|
||||
|
||||
## Opening a PR
|
||||
|
||||
|
||||
+3
-11
@@ -14,7 +14,7 @@
|
||||
# config.default.json, so partial overrides work. Absent file = defaults.
|
||||
|
||||
# ---- Build stage ----
|
||||
FROM node:26-bookworm-slim AS build
|
||||
FROM node:24-slim AS build
|
||||
|
||||
WORKDIR /opt/puter
|
||||
|
||||
@@ -59,7 +59,7 @@ RUN set -e; \
|
||||
wait $pjs_pid
|
||||
|
||||
# ---- Runtime stage (slim — no build tools) ----
|
||||
FROM node:26-bookworm-slim
|
||||
FROM node:24-slim
|
||||
|
||||
WORKDIR /opt/puter
|
||||
|
||||
@@ -73,24 +73,16 @@ COPY --from=build --chown=node:node /opt/puter .
|
||||
RUN mkdir -p /etc/puter /var/puter && \
|
||||
chown -R node:node /etc/puter /var/puter
|
||||
|
||||
# Fix outdated HOME_URL in /etc/os-release (see issue #4329)
|
||||
# The base image may contain a reference to the old phoenix repository.
|
||||
RUN if [ -f /etc/os-release ]; then \
|
||||
sed -i 's|HOME_URL="https://github.com/HeyPuter/phoenix"|HOME_URL="https://github.com/HeyPuter/puter"|' /etc/os-release; \
|
||||
fi
|
||||
|
||||
# Self-hosters mount their override at this exact path. The v2 loader
|
||||
# deep-merges it over config.default.json (see backend/index.ts).
|
||||
ENV PUTER_CONFIG_PATH=/etc/puter/config.json
|
||||
ENV NODE_OPTIONS=--enable-source-maps
|
||||
|
||||
EXPOSE 4100
|
||||
# The inbound SMTP receiver, when run from this image with the `smtp` profile.
|
||||
EXPOSE 2525
|
||||
|
||||
USER node
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=30s --retries=3 \
|
||||
CMD wget --no-verbose --tries=1 --spider http://puter.localhost:4100/test || exit 1
|
||||
|
||||
CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
|
||||
CMD ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/index.js"]
|
||||
@@ -10,10 +10,12 @@
|
||||
·
|
||||
<a href="https://apps.puter.com/">App Store</a>
|
||||
·
|
||||
<a href="https://builder.puter.com/">AI Builder</a>
|
||||
·
|
||||
<a href="https://developer.puter.com" target="_blank">Developers</a>
|
||||
·
|
||||
<a href="https://discord.com/invite/PQcx7Teh8u">Discord</a>
|
||||
·
|
||||
<a href="https://reddit.com/r/puter">Reddit</a>
|
||||
·
|
||||
<a href="https://twitter.com/HeyPuter">X</a>
|
||||
</p>
|
||||
|
||||
@@ -48,8 +50,6 @@ npm start
|
||||
```
|
||||
**→** This should launch Puter at http://puter.localhost:4100
|
||||
|
||||
To run this checkout with Docker, follow [Building from source](./doc/self-hosting.md#building-from-source-instead-of-pulling). Create a local `docker-compose.override.yml` to select the local build; keeping these settings out of `docker-compose.yml` avoids conflicts when pulling updates and keeps local configuration out of pull requests.
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
@@ -82,7 +82,10 @@ Puter is available as a hosted service at [**puter.com**](https://puter.com).
|
||||
Connect with the maintainers and community through these channels:
|
||||
|
||||
- Bug report or feature request? Please [open an issue](https://github.com/HeyPuter/puter/issues/new/choose).
|
||||
- Discord: [discord.com/invite/PQcx7Teh8u](https://discord.com/invite/PQcx7Teh8u)
|
||||
- X (Twitter): [x.com/HeyPuter](https://x.com/HeyPuter)
|
||||
- Reddit: [reddit.com/r/puter/](https://www.reddit.com/r/puter/)
|
||||
- Mastodon: [mastodon.social/@puter](https://mastodon.social/@puter)
|
||||
- Security issues or abuse reports? [[email protected]](mailto:[email protected])
|
||||
- Email maintainers at [[email protected]](mailto:[email protected])
|
||||
|
||||
@@ -117,9 +120,7 @@ This repository, including all its contents, sub-projects, modules, and componen
|
||||
- [Korean / 한국어](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.ko.md)
|
||||
- [Malay / Bahasa Malaysia](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.my.md)
|
||||
- [Malayalam / മലയാളം](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.ml.md)
|
||||
- [Dutch / Nederlands](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.nl.md)
|
||||
- [Polish / Polski](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.pl.md)
|
||||
- [Odia / ଓଡ଼ିଆ](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.od.md)
|
||||
- [Portuguese / Português](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.pt.md)
|
||||
- [Punjabi / ਪੰਜਾਬੀ](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.pa.md)
|
||||
- [Romanian / Română](https://github.com/HeyPuter/puter/blob/main/doc/i18n/README.ro.md)
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
# Reverse proxy in front of Puter — mirrors what the prod ALB does:
|
||||
# accepts every Host header, forwards to the Puter container, and lets
|
||||
# the Puter app handle subdomain-based routing internally (api.*,
|
||||
# site.*, app.*, dev.*, plus the per-user subdomains under those).
|
||||
#
|
||||
# To enable TLS:
|
||||
# 1. Drop a wildcard fullchain.pem + privkey.pem into ./puter/tls/
|
||||
# (see "Step 3 — TLS" in doc/self-hosting.md).
|
||||
# 2. Uncomment the `:443` block at the bottom of this file, and swap the
|
||||
# `:80` block for the redirect shown alongside it.
|
||||
# 3. Uncomment the `443:443` port mapping under the `caddy` service in
|
||||
# docker-compose.yml, and HTTPS_PORT in .env.
|
||||
# 4. Set `"protocol": "https"` and `"pub_port": 443` in config.json.
|
||||
{
|
||||
# Certs are supplied by the operator, not issued by Caddy. Puter serves
|
||||
# per-user sites and apps on dynamic subdomains (<name>.site.<domain>,
|
||||
# <name>.app.<domain>) — only a DNS-01 wildcard cert covers those, and
|
||||
# DNS-01 needs a provider plugin that isn't in the stock caddy image.
|
||||
# `off` stops Caddy attempting ACME on boot (it would fail and leave the
|
||||
# site unreachable) and stops it inventing its own http→https redirects.
|
||||
auto_https off
|
||||
}
|
||||
|
||||
# Shared handling, imported by the HTTP and HTTPS site blocks below so the
|
||||
# two can't drift apart.
|
||||
(puter_routes) {
|
||||
# Rough size cap that mirrors prod ALB defaults; tune for your uploads.
|
||||
# Puter chunks large uploads, so 1 GiB per request is plenty.
|
||||
request_body {
|
||||
max_size 1024MiB
|
||||
}
|
||||
|
||||
# RustFS — see the `s3` service in docker-compose.yml. Browsers PUT/GET
|
||||
# here for presigned-URL uploads / downloads. Matched on the `s3.`
|
||||
# subdomain of whatever the install domain is, so signature verification
|
||||
# works (Caddy preserves the original Host end-to-end) and HTTPS stays
|
||||
# clean — no mixed content from a port-9000 host publish.
|
||||
@s3 header_regexp Host ^s3\.
|
||||
handle @s3 {
|
||||
reverse_proxy s3:9000
|
||||
}
|
||||
|
||||
# Everything else, on every Host, goes to Puter — which routes on that
|
||||
# Host internally. Caddy forwards it unchanged and adds X-Forwarded-For
|
||||
# / -Proto / -Host, which is what `trust_proxy` in config.json counts.
|
||||
handle {
|
||||
reverse_proxy puter:4100 {
|
||||
# Stream responses through unbuffered — Puter uses SSE and
|
||||
# socket.io. WebSocket upgrades need no config of their own;
|
||||
# Caddy proxies them by default.
|
||||
flush_interval -1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ── HTTP (port 80) ─────────────────────────────────────────────────────
|
||||
# A site address with no hostname is the catch-all: it answers for every
|
||||
# Host, which is what Puter's subdomain routing requires.
|
||||
:80 {
|
||||
import puter_routes
|
||||
}
|
||||
|
||||
# ── HTTPS (port 443) — uncomment after dropping certs in ./puter/tls/ ──
|
||||
# Replace the `:80` block above with a redirect to force HTTPS everywhere:
|
||||
#
|
||||
# :80 {
|
||||
# redir https://{host}{uri} permanent
|
||||
# }
|
||||
#
|
||||
# :443 {
|
||||
# tls /etc/caddy/tls/fullchain.pem /etc/caddy/tls/privkey.pem
|
||||
# import puter_routes
|
||||
# }
|
||||
+1
-6
@@ -5,7 +5,7 @@
|
||||
"protocol": "http",
|
||||
"domain": "puter.localhost",
|
||||
"cookie_name": "puter_auth_token",
|
||||
"jwt_secret_v2": "dev-jwt-secret-v2-change-me",
|
||||
"jwt_secret": "dev-jwt-secret-change-me",
|
||||
"url_signature_secret": "dev-url-signature-secret-change-me",
|
||||
"allow_all_host_values": true,
|
||||
"allow_no_host_header": true,
|
||||
@@ -21,11 +21,6 @@
|
||||
"default_user_group": "78b1b1dd-c959-44d2-b02c-8735671f9997",
|
||||
"default_temp_group": "b7220104-7905-4985-b996-649fdcdb3c8f",
|
||||
"storage_capacity": 104857600,
|
||||
"disable_user_signup": false,
|
||||
"teams_enabled": false,
|
||||
"max_teams_per_user": 1,
|
||||
"max_seats_per_team_free": 3,
|
||||
"max_seats_per_team_paid": 40,
|
||||
"strict_email_verification_required": false,
|
||||
"gui_assets_root": "./src/gui",
|
||||
"puterjs_root": "./src/puter-js/dist",
|
||||
|
||||
+12
-271
@@ -7,20 +7,15 @@
|
||||
// Each setting lives at exactly one canonical key — there are no fallback
|
||||
// aliases. Values shown are illustrative, not production secrets.
|
||||
//
|
||||
// Keys consumed only by closed-source / hosted-prod extensions are
|
||||
// intentionally omitted.
|
||||
// Keys consumed only by closed-source / hosted-prod extensions (clickhouse,
|
||||
// cacheUpdateHandler, pages, prodMeteringAndBilling, …) are intentionally
|
||||
// omitted.
|
||||
|
||||
// ── Environment / identity ──────────────────────────────────────────
|
||||
"config_name": "template",
|
||||
// `dev` opens a browser on boot, skips blocked-email checks, and runs the
|
||||
// dev-time webpack watcher; `prod` serves pre-built bundles.
|
||||
"env": "dev",
|
||||
// Console output format. `json` replaces the global console so every call
|
||||
// emits one structured JSON line (level, timestamp, msg, and the active
|
||||
// request's trace id) — one event per call, so a line-oriented log
|
||||
// collector can't split stack traces across events and level filtering
|
||||
// works. Unset (the default) leaves console output human-readable.
|
||||
"log_format": "text",
|
||||
"version": "0.0.0",
|
||||
// Stable identity for this server node — used by pager alerts and
|
||||
// graceful-shutdown coordination.
|
||||
@@ -31,10 +26,6 @@
|
||||
"port": 4100,
|
||||
// Externally-visible port (set this when behind a reverse proxy on 80/443).
|
||||
"pub_port": 4100,
|
||||
// Idle keep-alive timeout in ms. Must stay above the idle timeout of any
|
||||
// proxy in front, or the proxy reuses connections this server has closed
|
||||
// and its clients see 502s.
|
||||
"keep_alive_timeout": 920000,
|
||||
"protocol": "http",
|
||||
"domain": "puter.localhost",
|
||||
// Fully-qualified externally-visible URL. Computed from protocol/domain/
|
||||
@@ -62,10 +53,6 @@
|
||||
// direct socket peer. NEVER set to `true` in prod — it trusts every hop
|
||||
// and makes X-Forwarded-For forgeable.
|
||||
"trust_proxy": false,
|
||||
// Request header a TLS-terminating proxy in front of the server sets to
|
||||
// the client's TLS fingerprint (JA3/JA4). Only set it when that proxy
|
||||
// overwrites any value the client sends. Unset disables it.
|
||||
// "tlsFingerprintHeader": "x-tls-fingerprint",
|
||||
"no_browser_launch": false,
|
||||
|
||||
// ── Dev watcher (devWatcher extension) ──────────────────────────────
|
||||
@@ -80,13 +67,10 @@
|
||||
|
||||
// ── Auth / session ──────────────────────────────────────────────────
|
||||
// ALWAYS replace these for any public install — `openssl rand -hex 64`.
|
||||
"jwt_secret_v2": "change-me",
|
||||
"jwt_secret": "change-me",
|
||||
"url_signature_secret": "change-me",
|
||||
"cookie_name": "puter_auth_token",
|
||||
"min_pass_length": 6,
|
||||
// When true, anonymous users must log in instead of creating temp or
|
||||
// permanent accounts.
|
||||
"disable_user_signup": false,
|
||||
"allow_system_login": false,
|
||||
"strict_email_verification_required": false,
|
||||
"captcha": {
|
||||
@@ -171,30 +155,17 @@
|
||||
|
||||
// ── Database ────────────────────────────────────────────────────────
|
||||
"database": {
|
||||
// `sqlite` for single-node/dev, `mysql` for MariaDB/MySQL,
|
||||
// or `postgres` for PostgreSQL.
|
||||
//
|
||||
// NOTE: `postgres` support is a community contribution and is not
|
||||
// exercised by Puter.com production. It boots, passes its own
|
||||
// integration tests against pgmock, and runs the common user/app/
|
||||
// fsentry/session/permission/OIDC flows — but less-traveled code
|
||||
// paths may surface MySQL/SQLite-isms that haven't been ported yet.
|
||||
// Expect rough edges and please file issues if you hit one. For
|
||||
// production self-hosting today, `mysql` (MariaDB) and `sqlite` are
|
||||
// the supported defaults.
|
||||
// `sqlite` for single-node / dev; `mysql` for self-host with MariaDB.
|
||||
"engine": "sqlite",
|
||||
// sqlite — file path on disk
|
||||
"path": "volatile/runtime/puter-database.sqlite",
|
||||
"targetVersion": 0,
|
||||
// mysql/postgres — connection details. PostgreSQL defaults to port
|
||||
// 5432 when `engine` is `postgres`; MySQL/MariaDB normally use 3306.
|
||||
// mysql — connection details
|
||||
"host": "",
|
||||
"port": 3306,
|
||||
"user": "",
|
||||
"password": "",
|
||||
"database": "",
|
||||
// postgres may also use a URL instead of host/user/password fields:
|
||||
// "connectionString": "postgres://puter:secret@localhost:5432/puter",
|
||||
// Optional read-replica pool. Reads route here when populated.
|
||||
"replica": {
|
||||
"host": "",
|
||||
@@ -203,10 +174,9 @@
|
||||
"password": "",
|
||||
"database": ""
|
||||
}
|
||||
// mysql/postgres self-host bootstrap: set `migrationPaths` to apply the
|
||||
// bundled schema on first boot. Idempotent — safe to leave on.
|
||||
// mysql self-host bootstrap: set `migrationPaths` to apply the bundled
|
||||
// schema on first boot. Idempotent — safe to leave on.
|
||||
// "migrationPaths": ["./src/backend/clients/database/migrations/mysql"]
|
||||
// "migrationPaths": ["./src/backend/clients/database/migrations/postgres"]
|
||||
},
|
||||
|
||||
// ── DynamoDB (KV store) ─────────────────────────────────────────────
|
||||
@@ -219,9 +189,7 @@
|
||||
// "bootstrapTables": true,
|
||||
"path": "",
|
||||
// Credentials. NOTE: snake_case here, unlike `s3.s3Config` below.
|
||||
// For dynamodb-local, any non-empty values work. On AWS,
|
||||
// `"useCredentialChain": true` uses the default credential chain
|
||||
// instead of keys.
|
||||
// For dynamodb-local, any non-empty values work.
|
||||
"aws": {
|
||||
"access_key": "",
|
||||
"secret_key": "",
|
||||
@@ -258,168 +226,6 @@
|
||||
"user": "",
|
||||
"pass": ""
|
||||
}
|
||||
// Each send names a stream: "auth", "signup" or "notify". A stream
|
||||
// listed here adds its headers to every message on it, and its "from"
|
||||
// when the sender gives none. Tags are written to "tagsHeader".
|
||||
// "streams": {
|
||||
// "notify": {
|
||||
// "from": "\"Puter\" <[email protected]>",
|
||||
// "headers": { "X-Example-Stream": "notify" },
|
||||
// "tagsHeader": "X-Example-Tags"
|
||||
// }
|
||||
// }
|
||||
},
|
||||
|
||||
// ── Inbound mail (separate process) ─────────────────────────────────
|
||||
// Mailboxes addressed at one of your own domains. `secret` gates the
|
||||
// ingress endpoint that files a message into the recipient's ~/.mail.
|
||||
//
|
||||
// `localServer` additionally runs an SMTP listener that accepts mail from
|
||||
// other mail servers and hands each message to that endpoint. It runs as
|
||||
// its own process - `npm run start:smtp`, or the `smtp` compose profile -
|
||||
// and stores nothing itself. Without it, nothing delivers to the endpoint
|
||||
// unless you point something else at it.
|
||||
//
|
||||
// The listener is unauthenticated by nature: anything on the internet may
|
||||
// hand it a message for an address in `localDomains`. It performs no spam,
|
||||
// SPF, DKIM or DMARC checks. Read the self-hosting guide before exposing
|
||||
// it.
|
||||
// "userEmail": {
|
||||
// "secret": "a long random string",
|
||||
// "localServer": true,
|
||||
// "localDomains": ["example.com"],
|
||||
// // Container-internal by default; publish 25 onto it from outside.
|
||||
// "localPort": 2525,
|
||||
// // Defaults to <api_base_url>/email/ingress. The endpoint is served
|
||||
// // on the `api` subdomain, so when this points somewhere internal,
|
||||
// // `localIngressHost` has to name that virtual host.
|
||||
// "localIngressUrl": "http://puter:4100/email/ingress",
|
||||
// "localIngressHost": "api.example.com",
|
||||
// "localHostname": "mx.example.com",
|
||||
// "localMaxRecipients": 50,
|
||||
// // Concurrent connections. Each message is held in memory while it
|
||||
// // is delivered, so this also bounds memory use.
|
||||
// "localMaxClients": 20
|
||||
// },
|
||||
|
||||
// ── Sharing ─────────────────────────────────────────────────────────
|
||||
// Email a recipient who already has an account about a new share. On
|
||||
// unless set to false; they opt out via the unsubscribe link, or by
|
||||
// blocking a sender. An invite to an address with no account always goes.
|
||||
// "share_email_notifications": false,
|
||||
//
|
||||
// New shares one account may create per UTC day. Default 200.
|
||||
// "share_daily_limit": 200,
|
||||
//
|
||||
// How often a share may interrupt its recipient — the notification pushed
|
||||
// to their screen and the email with it. Over budget the share still
|
||||
// succeeds and their notification is still brought up to date; only the
|
||||
// interruption is dropped. The recipient bounds are what stop many senders
|
||||
// burying one person between them. A non-positive value removes a bound.
|
||||
// "share_notify_limits": {
|
||||
// "pairWindowSeconds": 900,
|
||||
// "pairDaily": 20,
|
||||
// "recipientHourly": 10,
|
||||
// "recipientDaily": 50,
|
||||
// "emailBatchSeconds": 90
|
||||
// },
|
||||
|
||||
// ── Teams / teams ──────────────────────────────────────────────
|
||||
// Off by default, and off is the state every existing install stays in.
|
||||
// The tables ship either way and sit inert; this gates whether `/teams`
|
||||
// is registered at all, so with it off the paths do not exist rather than
|
||||
// existing and refusing. It is also the backout: turning it off removes
|
||||
// the feature without touching data. Teams ships as an extension; on,
|
||||
// the server refuses to boot unless one registers it.
|
||||
"teams_enabled": false,
|
||||
//
|
||||
// Live teams one user may own. Soft-deleted ones do not count, so
|
||||
// deleting yours frees the slot. Default 1.
|
||||
"max_teams_per_user": 1,
|
||||
//
|
||||
// Seats one team may provision. Each seat is a real Puter account taking a
|
||||
// name from the global username pool, so this is what bounds a team's blast
|
||||
// radius. The owner's plan decides which of the two applies; defaults are 3
|
||||
// and 40. A plan registered with `teamSeatCap` uses that instead. Lowering
|
||||
// a cap below a team's current seat count blocks new provisioning and
|
||||
// disables nobody. Deployments upgrading from the old flat default of 50:
|
||||
// without a billing extension every owner reads as free, so set
|
||||
// `max_seats_per_team` to keep the old behavior.
|
||||
"max_seats_per_team_free": 3,
|
||||
"max_seats_per_team_paid": 40,
|
||||
//
|
||||
// One flat cap for every team, whatever the owner pays. Setting it
|
||||
// overrides both of the above, so a deployment that does not sell seats can
|
||||
// ignore plans entirely. Unset by default — leave it that way to keep the
|
||||
// free cap meaningful.
|
||||
// "max_seats_per_team": 50,
|
||||
//
|
||||
// Staged rollout: only these email domains may create a team, and the tab
|
||||
// renders for them even with `gui_params.teams_ui` off. Members of an
|
||||
// existing team always pass. Unset means everyone, gated by `teams_ui`.
|
||||
// "teams_allowed_email_domains": ["puter.com"],
|
||||
|
||||
// ── Notifications ───────────────────────────────────────────────────
|
||||
// How long a notification is kept, in days from creation. Acknowledged or
|
||||
// not, a row past this is swept. Off by default; any value above 0 arms
|
||||
// the sweep.
|
||||
// "notificationRetentionDays": 14,
|
||||
//
|
||||
// Ms a live notification waits for open tabs to acknowledge it.
|
||||
// "notificationAckTimeoutMs": 5000,
|
||||
|
||||
// ── Alarms / alerting ───────────────────────────────────────────────
|
||||
// Where system alarms go. Severity is the routing decision — each
|
||||
// transport takes everything at or above its own `minSeverity`:
|
||||
//
|
||||
// critical — an unhandled server error; pages on-call.
|
||||
// error — pages as well; prefer critical or warning.
|
||||
// warning — look at it today; no page.
|
||||
// info — a record in the chat channel only.
|
||||
//
|
||||
// Both transports are off unless enabled, so a self-hosted node just
|
||||
// logs its alarms to the console.
|
||||
"pager": {
|
||||
// Severity for call sites that don't pick one. Default "critical".
|
||||
"defaultSeverity": "critical",
|
||||
|
||||
// Retier or silence an alarm without a deploy. Keys are alarm ids,
|
||||
// or a prefix ending in `*`; the exact id wins over a pattern, and
|
||||
// the longest matching pattern wins among patterns. Values are a
|
||||
// severity or "mute". This is applied last, so it overrides both the
|
||||
// call site and any known-error rule.
|
||||
"severityOverrides": {
|
||||
// "cronMonitor:*": "info",
|
||||
// "http_500:GET:/some/flapping/route:*": "mute"
|
||||
},
|
||||
|
||||
"pagerduty": {
|
||||
"enabled": false,
|
||||
"routingKey": "",
|
||||
// Lowest severity that reaches PagerDuty. Default "warning",
|
||||
// which keeps `info` out of the paging system entirely.
|
||||
"minSeverity": "warning"
|
||||
},
|
||||
|
||||
// Slack incoming webhook — the low-noise destination for everything
|
||||
// that shouldn't page.
|
||||
"slack": {
|
||||
"enabled": false,
|
||||
"webhookUrl": "",
|
||||
// Optional; defaults to the channel the webhook was created for.
|
||||
"channel": "#alerts",
|
||||
"username": "puter-alarms",
|
||||
// Severity window posted to Slack. The ceiling defaults to
|
||||
// "info" when PagerDuty is configured — what pages belongs in
|
||||
// the pager, not in chat — and to "critical" when Slack is the
|
||||
// only transport.
|
||||
"minSeverity": "info",
|
||||
"maxSeverity": "info",
|
||||
// Don't repost the same alarm id within this window. The first
|
||||
// occurrence always posts; the next post that gets through
|
||||
// reports how many occurrences piled up. 0 disables throttling.
|
||||
"repeatThrottleMs": 900000
|
||||
}
|
||||
},
|
||||
|
||||
// ── Rate limiting ───────────────────────────────────────────────────
|
||||
@@ -436,52 +242,16 @@
|
||||
// ─ Chat / completion ─
|
||||
"claude": { "apiKey": "" },
|
||||
"openai-completion": { "apiKey": "" },
|
||||
"azure-openai": {
|
||||
"apiKey": "",
|
||||
"apiURL": ""
|
||||
},
|
||||
"gemini": { "apiKey": "" },
|
||||
// Meta Model API (Muse Spark). `muse-spark-1.2-contributor`
|
||||
// trades a ~12x discount for Meta training on its prompts and
|
||||
// completions, so it is only served to callers that name it.
|
||||
"meta": { "apiKey": "" },
|
||||
"groq": { "apiKey": "" },
|
||||
"deepseek": { "apiKey": "" },
|
||||
"mistral": { "apiKey": "" },
|
||||
"xai": { "apiKey": "" },
|
||||
"moonshot": { "apiKey": "" },
|
||||
// MiniMax. One key powers chat and video generation (H3, Hailuo);
|
||||
// video calls go to the same host, beside the chat API's `/v1`.
|
||||
"minimax": {
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://api.minimax.io/v1"
|
||||
},
|
||||
"openrouter": {
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://openrouter.ai/api/v1"
|
||||
},
|
||||
"infron": {
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://llm.onerouter.pro/v1"
|
||||
},
|
||||
// BytePlus ModelArk. One key powers chat (Seed/GLM/DeepSeek), image
|
||||
// generation (Seedream) and video generation (Seedance); `apiBaseUrl`
|
||||
// selects the region; see
|
||||
// https://docs.byteplus.com/en/docs/ModelArk/1330310 for options.
|
||||
"byteplus": {
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://ark.ap-southeast.bytepluses.com/api/v3"
|
||||
},
|
||||
// Kling AI. One API key (not the legacy AccessKey/SecretKey pair)
|
||||
// powers video generation, image generation and TTS; `apiBaseUrl`
|
||||
// defaults to https://api-singapore.klingai.com.
|
||||
"kling": { "apiKey": "" },
|
||||
"zai": { "apiKey": "" },
|
||||
"alibaba": { "apiKey": "" },
|
||||
"hoonify": {
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://api.hoonify.ai/v1"
|
||||
},
|
||||
"together-ai": { "apiKey": "" },
|
||||
// Local Ollama. `enabled: false` skips the auto-probe at startup
|
||||
// (otherwise Puter logs ECONNREFUSED on every boot when no Ollama
|
||||
@@ -503,6 +273,7 @@
|
||||
"xai-image-generation": { "apiKey": "" },
|
||||
|
||||
// ─ Video generation ─
|
||||
"openai-video-generation": { "apiKey": "" },
|
||||
"together-video-generation": { "apiKey": "" },
|
||||
"gemini-video-generation": { "apiKey": "" },
|
||||
|
||||
@@ -514,14 +285,11 @@
|
||||
"defaultVoiceId": "",
|
||||
"speechToSpeechModelId": ""
|
||||
},
|
||||
// Both AWS providers also take `"useCredentialChain": true` in place
|
||||
// of keys, like `dynamo.aws`.
|
||||
"aws-polly": {
|
||||
"access_key": "",
|
||||
"secret_key": "",
|
||||
"region": "us-west-2"
|
||||
},
|
||||
"speechify": { "apiKey": "" },
|
||||
"aws-textract": {
|
||||
"access_key": "",
|
||||
"secret_key": "",
|
||||
@@ -579,33 +347,6 @@
|
||||
],
|
||||
|
||||
// ── Metering ────────────────────────────────────────────────────────
|
||||
// When true, every account resolves to an unlimited policy: usage is still
|
||||
// recorded, but nothing is ever refused for lack of budget. This is the
|
||||
// setting for a deployment with no way to buy more — without it, accounts
|
||||
// are held to the free monthly allowance and start getting 402s from the AI
|
||||
// surfaces, file transfers and KV once they pass it.
|
||||
"unlimitedMetering": false,
|
||||
|
||||
// Whether an account that has spent its whole allowance is refused the
|
||||
// operations that spend it — file transfers, KV calls. Recording is
|
||||
// unaffected either way. `workers` extends the same refusal to
|
||||
// worker-driven calls, which are exempt by default because a deployed
|
||||
// worker has nowhere to surface a payment prompt. `subscriptions` covers
|
||||
// the separate question of surfaces reserved for paid plans (the
|
||||
// vendor-compatible AI endpoints): turn it off on a deployment with no
|
||||
// paid plans, or every account is refused them.
|
||||
// "meteringEnforcement": { "enabled": true, "workers": false, "subscriptions": true },
|
||||
|
||||
// Fleet-wide spend rate, in micro-cents per minute, past which metering
|
||||
// raises the `metering:excessiveGlobalUsageRate` alarm. Omit it (the
|
||||
// default) to leave the check off: the only useful value is a multiple of
|
||||
// what this deployment's normal traffic costs, so it has to be measured
|
||||
// rather than guessed, and a stale number here alarms on healthy growth.
|
||||
// "maxGlobalUsagePerMinute": 200000000
|
||||
|
||||
// How many full months past its own a monthly usage record is kept before
|
||||
// it expires. Defaults to 3; set to 0 to keep every month's records
|
||||
// forever. Changing it, including to 0, applies to records first written
|
||||
// after the change; records already stamped keep their expiry.
|
||||
// "meteringRetentionMonths": 3
|
||||
// When true, all metering checks pass — no per-actor limits enforced.
|
||||
"unlimitedMetering": false
|
||||
}
|
||||
-145
@@ -1,145 +0,0 @@
|
||||
# Alarms
|
||||
|
||||
`clients.alarm` is the one way code reports that something is wrong. It
|
||||
de-dupes by alarm id, counts occurrences, and routes each alarm to alert
|
||||
transports by **severity**.
|
||||
|
||||
```ts
|
||||
this.clients.alarm.create(
|
||||
`metering_write_failed:${userUuid}`, // de-dupe key
|
||||
'Metering write failed', // what a human reads
|
||||
{ userUuid, appId, error }, // context fields
|
||||
'info', // severity
|
||||
);
|
||||
```
|
||||
|
||||
## Severity is the routing decision
|
||||
|
||||
| Severity | Meaning | Goes to |
|
||||
| ---------- | ------------------------------------------------------ | ----------- |
|
||||
| `critical` | An unhandled server error. Someone gets woken up. | Pager |
|
||||
| `error` | Same urgency as critical; prefer one of the other two. | Pager |
|
||||
| `warning` | Worth a look today. Nobody is paged. | Pager (low) |
|
||||
| `info` | A record of something expected-but-notable. | Chat |
|
||||
|
||||
Each transport declares the severity window it accepts, so the value a call
|
||||
site passes is what decides where the alarm lands. The two windows don't
|
||||
overlap by default: anything that pages lives in the paging system, and chat
|
||||
is the record of what didn't. The bar for `critical` is
|
||||
deliberately high: an unhandled 5xx out of the HTTP error handler is the main
|
||||
thing that still pages. Anything a human can look at tomorrow is `warning`,
|
||||
and anything that's just worth recording is `info`.
|
||||
|
||||
Omitting the severity takes `pager.defaultSeverity` (itself `critical`), so
|
||||
pass one explicitly unless you really mean "page someone".
|
||||
|
||||
### Choosing one
|
||||
|
||||
- Did the server fail to do its job in a way nobody expected? → `critical`
|
||||
- Is a background job, rate, or dependency degraded, or has an account spent
|
||||
far past its credits? → `warning`
|
||||
- Is this a user doing something notable (tripping an abuse heuristic)?
|
||||
→ `info`
|
||||
- Did one of *our own* limits reject a caller — a rate limit, a concurrency
|
||||
cap, a quota? → don't alarm at all. The limit doing its job is not an
|
||||
event; the 429 is the whole signal, and alarming on it only produces noise
|
||||
proportional to traffic. An *upstream provider* rate-limiting us is the
|
||||
opposite case and still alarms (`upstream_rate_limited`, `info`) — that one
|
||||
is not something we chose. The exception is a free model: nothing is billed,
|
||||
nothing is actionable, and the throttling is the price of the model, so the
|
||||
AI chat driver marks those `noAlarm` and only paid models still record.
|
||||
|
||||
`noAlarm` is the code-level mute: an `HttpError` carrying it skips the
|
||||
terminal gate entirely, no matter its status or code. It is for failures the
|
||||
call site *already knows* are expected and traffic-proportional — reach for it
|
||||
there, not as a way to quiet an alarm you haven't diagnosed
|
||||
(`severityOverrides` below is the knob for that).
|
||||
|
||||
An extension whose signals are all one tier can default its own local
|
||||
`raiseAlarm` helper to that tier instead of repeating it at every call site —
|
||||
see [extensions/cronMonitor](../../../extensions/cronMonitor/index.js).
|
||||
|
||||
## One incident per occurrence, unless you say otherwise
|
||||
|
||||
Alarms always de-dupe *in process* — repeats of an id bump its occurrence
|
||||
count rather than creating a second alarm. What that means for the pager is a
|
||||
separate decision, and by default every occurrence opens its own PagerDuty
|
||||
incident: two failed scans an hour apart are two things that happened, and
|
||||
closing one shouldn't hide the other.
|
||||
|
||||
Pass `{ dedup: true }` as a fifth argument when repeats of the id really are
|
||||
one recurring fault, and they collapse onto a single incident carrying the
|
||||
occurrence count:
|
||||
|
||||
```ts
|
||||
this.clients.alarm.create(alarmId, message, fields, 'critical', {
|
||||
dedup: true,
|
||||
});
|
||||
```
|
||||
|
||||
The HTTP error handler uses it: its id is route + error signature, so a hot
|
||||
loop of the same crash is one incident with N occurrences instead of N pages.
|
||||
Dedup and Slack's repeat throttle mean a responder sees only the latest
|
||||
occurrence, so the handler attaches the `HttpError`'s `fields` to the alarm as
|
||||
`details` — put what a responder needs (which upstreams failed, and how)
|
||||
there. Everything in `fields` is also returned to the client in the response
|
||||
body, so it has to be safe to show the caller.
|
||||
Reach for it anywhere else only when the id is that specific — otherwise a
|
||||
per-request alarm can flood the pager.
|
||||
|
||||
## Configuration
|
||||
|
||||
Everything lives under `pager` in config (see
|
||||
[config.template.jsonc](../config.template.jsonc) for the annotated version).
|
||||
Both transports are off unless enabled, so a self-hosted node just logs
|
||||
alarms to the console.
|
||||
|
||||
```jsonc
|
||||
"pager": {
|
||||
"defaultSeverity": "critical",
|
||||
"severityOverrides": { "cronMonitor:*": "info" },
|
||||
"pagerduty": { "enabled": true, "routingKey": "…", "minSeverity": "warning" },
|
||||
"slack": {
|
||||
"enabled": true,
|
||||
"webhookUrl": "…",
|
||||
"channel": "#alerts",
|
||||
"minSeverity": "info",
|
||||
"maxSeverity": "info",
|
||||
"repeatThrottleMs": 900000,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Slack's `maxSeverity` defaults to `info` whenever PagerDuty is configured, and
|
||||
to `critical` when it isn't — a node with only a webhook still sees
|
||||
everything. Raise it to have chat mirror the paging tiers as well.
|
||||
|
||||
### Retiering without a deploy
|
||||
|
||||
`severityOverrides` is the escape hatch for an alarm that turns out to be
|
||||
noisier or more serious than its call site assumed. Keys are alarm ids or a
|
||||
prefix ending in `*`; the exact id beats a pattern, and the longest matching
|
||||
prefix wins among patterns. Values are a severity, or `mute` to drop the
|
||||
alarm before any transport sees it.
|
||||
|
||||
It is applied *after* the call site's severity and any known-error rule, so
|
||||
config always has the last word.
|
||||
|
||||
### Repeat throttling
|
||||
|
||||
Two levels, and they stack.
|
||||
|
||||
The client itself backs off before a repeat is reported at all: every
|
||||
occurrence is reported up to a small burst, then one per interval. A fault
|
||||
recurring hundreds of times a second still counts every occurrence, but it
|
||||
does not write a log line or build an alert payload for each one.
|
||||
|
||||
The count lasts only as long as the client remembers the alarm. One that
|
||||
hasn't recurred for 24 hours is forgotten, and the client holds at most 1000
|
||||
at once, dropping the least recently raised first. Either way the next
|
||||
occurrence is a new alarm, with its count and burst starting over.
|
||||
|
||||
Below that, the chat transport won't repost the same alarm id within
|
||||
`repeatThrottleMs` (default 15 minutes). The first occurrence always posts,
|
||||
and the next one that gets through reports how many piled up in between —
|
||||
so a hot loop reads as one message with a count, not a wall of them.
|
||||
+16
-27
@@ -1,6 +1,6 @@
|
||||
# Backend Architecture
|
||||
|
||||
Loosely inspired by the Controller–Service–Repository pattern with dependency injection. The backend is a stack of layers; each depends only on the layers beneath it and receives them through its constructor from `PuterServer` ([src/backend/server.ts](../src/backend/server.ts)), which instantiates the layers in order. The rules for working within the layers are in [AGENTS.md](../AGENTS.md#backend).
|
||||
Loosely inspired by the Controller–Service–Repository pattern with dependency injection. The backend is organized as a stack of layers where each layer only depends on the layers beneath it, and `PuterServer` ([src/backend/server.ts](../src/backend/server.ts)) instantiates each layer in order and hands the instances down to the next.
|
||||
|
||||
## Layers
|
||||
|
||||
@@ -31,6 +31,8 @@ block-beta
|
||||
style CFG fill:#334155,stroke:#1e293b,color:#fff
|
||||
```
|
||||
|
||||
Each layer only depends on the layers beneath it, and every dependency is injected through the constructor by `PuterServer`. Extensions sit alongside this stack and can register into any layer — see [Extensions](#extensions) below.
|
||||
|
||||
| Layer | Lives in | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| **Controllers** | [src/backend/controllers/](../src/backend/controllers/) | Route handlers. Parse + validate input, apply per-route gates (auth, subdomain, rate limit, body parsers — see `RouteOptions`), call into services, format responses. |
|
||||
@@ -40,7 +42,7 @@ block-beta
|
||||
| **Clients** | [src/backend/clients/](../src/backend/clients/) | Adapters for external/internal services (sql, redis, s3, dynamodb, email, event bus, …). Knows protocols, not domain concepts. |
|
||||
| **Config** | `config.*.json` → `IConfig` | The flat, typed config object every layer receives at construction. |
|
||||
|
||||
Extensions sit alongside this stack and can register into any layer; see [Extensions](#extensions).
|
||||
Each layer receives the layers beneath it through its constructor, so dependencies are explicit and traceable from `PuterServer`. A controller does not reach into a client directly; if it needs one, the right move is usually a service.
|
||||
|
||||
## Entry point: `PuterServer`
|
||||
|
||||
@@ -53,25 +55,17 @@ Extensions sit alongside this stack and can register into any layer; see [Extens
|
||||
|
||||
## Context (ALS)
|
||||
|
||||
[`Context`](../src/backend/core/context.ts), backed by `AsyncLocalStorage`, carries per-request state without threading it through every function signature. The request-context middleware opens a scope per request after the auth probe runs, so anything inside a request handler can call `Context.get('actor')` / `Context.get('req')`. It's used sparingly, mostly for `actor` and `req`; see the [Context rule](../AGENTS.md#backend).
|
||||
We use [`Context`](../src/backend/core/context.ts) — backed by `AsyncLocalStorage` — to carry per-request state without threading it through every function signature. It is used **sparingly**, mostly for `actor` and `req`. The request-context middleware opens a scope per request after the auth probe runs; anything inside a request handler can call `Context.get('actor')` / `Context.get('req')` instead of plumbing it as an argument.
|
||||
|
||||
## Actors
|
||||
|
||||
An [`Actor`](../src/backend/core/actor.ts) is who a request is acting as. Its two app fields are not interchangeable: `app` is the app the actor carries *directly* (an app-under-user token), while `effectiveApp` is the app it ultimately acts as — its own, or the one that issued its access token. An access-token actor has no `app` of its own, so a gate that reads `app` answers "no app" for it and falls open.
|
||||
|
||||
`makeActor` resolves `effectiveApp` once at construction, and `assertResolvedActor` at the request edge rejects any actor that skipped it — so reading `effectiveApp` needs no fallback. Reach for `app` only where the direct app genuinely is the question, and say why in a comment.
|
||||
|
||||
- `actor.effectiveApp` — which app is acting. This is the default; nearly every scoping rule, namespace key and attribution wants it.
|
||||
- `isAppActor(actor)` — a direct app actor specifically. Use it where a token must *not* be treated as its issuer, e.g. the permission scanners that read an app's own grant rows.
|
||||
- `isPlainUserActor(actor)` — the account acting through nothing at all; `isAccountContext(actor)` for the wider "the account's own reach", which also admits a full-access token.
|
||||
Prefer explicit arguments. Reach for `Context` only when the value is truly request-scoped and would otherwise need to thread through many layers.
|
||||
|
||||
## Extensions
|
||||
|
||||
Extensions live in [extensions/](../extensions/), parallel the layered stack, and hold the parts of the system Puter still works without. [AGENTS.md](../AGENTS.md#extensions) has the rule for what belongs in one.
|
||||
Extensions live alongside core ([packages/puter/extensions/](../extensions/)) and parallel the layered stack. They are meant for **non-crucial parts of the system** — things Puter still works without if removed.
|
||||
|
||||
- **Good extensions**: [thumbnails](../extensions/thumbnails.ts), [serverInfo](../extensions/serverInfo.ts), [devWatcher](../extensions/devWatcher.ts) — opt-in features cleanly bolted on.
|
||||
- **Should probably be core**: [metering](../extensions/metering.ts), [appTelemetry](../extensions/appTelemetry.ts) — clients now expect these to be present, so the "extension" framing is misleading.
|
||||
- **Shouldn't have been an extension**: [whoami](../extensions/whoami.ts) — it's load-bearing for every authenticated client.
|
||||
- **Shouldn't have been an extension**: [whoami](../extensions/whoami.ts) — it's load-bearing for every authenticated client. Keep this one in mind as a cautionary example when deciding whether something belongs in an extension.
|
||||
|
||||
### Extension API
|
||||
|
||||
@@ -94,22 +88,17 @@ import { extension } from '@heyputer/backend/src/extensions';
|
||||
const services = extension.import('service');
|
||||
|
||||
extension.get('/healthcheck/deep', { subdomain: 'api', adminOnly: true }, async (_req, res) => {
|
||||
res.json(await services.health.getStatus());
|
||||
res.json({ ok: await services.health.runDeepCheck() });
|
||||
});
|
||||
|
||||
extension.on('puter.signup.success', (_key, data) => {
|
||||
console.log('new user', data.username);
|
||||
extension.on('user.signup', (_key, data) => {
|
||||
console.log('new user', data.user.username);
|
||||
});
|
||||
```
|
||||
|
||||
The `app.recommended` event lets extensions customize desktop recommendations.
|
||||
Its `appNames` array starts as a fresh copy of the default ordered names on each
|
||||
call. Listeners may mutate or replace it, including setting it to an empty array.
|
||||
Async listeners are awaited before names are resolved to app summaries; names
|
||||
that do not exist are skipped.
|
||||
## Conventions
|
||||
|
||||
```ts
|
||||
extension.on('app.recommended', (_key, data) => {
|
||||
data.appNames = ['editor', 'camera'];
|
||||
});
|
||||
```
|
||||
- **TypeScript preferred** in new code where feasible. Existing JS is fine; convert opportunistically when you're already touching a file.
|
||||
- **`camelCase`** for variable/function names; **`PascalCase`** for classes and for files that contain a class (`AuthService.ts`, `KVStoreDriver.ts`).
|
||||
- **Deduplicate**. If two services need the same logic, lift it into a util/helper rather than calling sideways across the same layer — services should not depend on other services for code reuse.
|
||||
- **Don't reach across layers.** Controllers do not poke clients directly; services do not register routes. If you find yourself wanting to, that's usually a signal the abstraction is wrong.
|
||||
@@ -1,106 +0,0 @@
|
||||
# Contributing APIs
|
||||
|
||||
How to add a new public API to Puter, and how to maintain one that already exists. "Public API" means anything applications can call: HTTP endpoints, `/drivers/*` methods, and the puter.js methods that wrap them. This guide is written for human contributors and AI agents alike — [AGENTS.md](../AGENTS.md) defers to it for API work.
|
||||
|
||||
Companion docs: [architecture.md](architecture.md) (backend layering), [pagination.md](pagination.md) (list APIs), [src/puter-js/tests/api/README.md](../src/puter-js/tests/api/README.md) (SDK test environment).
|
||||
|
||||
## Rule zero: don't break callers
|
||||
|
||||
puter.js is served live from `https://js.puter.com/v2/` with no version pinning — every app in existence picks up your change the moment it deploys, and the backend endpoints underneath have the same property. Assume every observable behavior (parameter handling, response fields, error codes, ordering) has someone depending on it.
|
||||
|
||||
Every change is backward compatible unless a maintainer has explicitly agreed to a break beforehand.
|
||||
|
||||
## Core or extension?
|
||||
|
||||
The first decision is where the API lives. If **nothing in core will call it, prefer an extension** over wiring it into core. [AGENTS.md](../AGENTS.md#extensions) has the rule and its test; [architecture.md](architecture.md#extension-api) documents the `extension` API.
|
||||
|
||||
## Adding a new API
|
||||
|
||||
Work through all seven steps; the PR is complete when every one is.
|
||||
|
||||
### 1. Design the surface first
|
||||
|
||||
- Sketch the signature, options, return shape, and error cases before writing code. Find the two or three most similar existing APIs and match their conventions.
|
||||
- New parameter and field names follow the [naming rule](../AGENTS.md#language--files) (`camelCase`).
|
||||
- Anything returning a list follows the [pagination convention](pagination.md): `limit`/`cursor` in, `{ items, cursor, total }` envelope out.
|
||||
- Prefer an options object over a growing list of positional parameters, but keep the common case callable with a single argument where siblings do.
|
||||
|
||||
### 2. Backend
|
||||
|
||||
Follow the layered stack ([architecture.md](architecture.md), [layer rules](../AGENTS.md#backend)): a controller or driver at the edge, business logic in a service, persistence in a store. Return exactly what the caller needs and no more — every response field you ship is permanent — and use stable `snake_case` error codes.
|
||||
|
||||
#### Controller or driver?
|
||||
|
||||
Both are supported ways to define an API. **Prefer a controller when you need fine-grained control** — URL shape, HTTP verbs, per-route gates, response and streaming formats. A **driver** fits when the API is a set of RPC methods implementing one of the named interfaces on `/drivers/*` (`puter-kvstore`, `puter-chat-completion`, …) and the generic driver plumbing — one call envelope, interchangeable implementations, per-method policies — covers what you need.
|
||||
|
||||
- **Controller:** extend `PuterController` ([src/backend/controllers/types.ts](../src/backend/controllers/types.ts)) and declare routes with the `@Controller(prefix)` class decorator plus `@Get`/`@Post`/… method decorators ([src/backend/core/http/decorators.ts](../src/backend/core/http/decorators.ts)), each taking `(path, routeOptions)` — or override `registerRoutes(router)` imperatively. Core controllers register in [src/backend/controllers/index.ts](../src/backend/controllers/index.ts); extensions use `extension.registerController(...)` or the plain route helpers.
|
||||
- **Driver:** extend `PuterDriver` ([src/backend/drivers/types.ts](../src/backend/drivers/types.ts)) and mark it with the `@Driver(interfaceName, opts)` decorator ([src/backend/drivers/decorators.ts](../src/backend/drivers/decorators.ts)), which also declares its policies. Core drivers register in [src/backend/drivers/index.ts](../src/backend/drivers/index.ts); extensions use `extension.registerDriver(...)`.
|
||||
|
||||
#### Middleware and gates
|
||||
|
||||
- **Controller routes** (and extension routes — same options) take [`RouteOptions`](../src/backend/core/http/types.ts): auth gates (`requireAuth`, `requireUserActor`, `noUserSession`, `adminOnly`, `allowedAppIds`, and the access-token controls), `subdomain` routing, per-route `rateLimit`, body parsers, and arbitrary extra `middleware`. The auth flavors are subtle and default-deny — read the JSDoc on each field before picking.
|
||||
- **Driver methods** get their policies from the `@Driver` options: per-method `rateLimit` (limit/window/backend), `concurrent` in-flight caps (optionally `bySubscription`), `requireSubscription`, `requireReputation`, and `noUserSession`. The `/drivers/call` surface enforces them.
|
||||
- **Paying accounts only:** `requireSubscription`, as a route option on a controller endpoint or in the per-method `@Driver({ requireSubscription })` block on a driver (`/drivers/call` is one shared route, so a driver's requirement can't live in route options). `true` accepts any plan that isn't free, so a plan an extension registers counts without core naming it; an array of policy ids (`['business', 'pro']`) accepts only those; `false` or leaving it out means no requirement. An empty array is a boot error, since it would read as subscribers-only while admitting everyone. The check is a map lookup on the metering service's cached per-actor subscription. It asks which plan the account is on; `requireCredits` asks whether it has budget left. Deployments with no paid plans (self-hosted installs) turn it off with `meteringEnforcement.subscriptions: false`.
|
||||
- **Trusted-enough accounts only:** `requireReputation`, as a route option or in the per-method `@Driver({ requireReputation })` block. Opt-in; implies `requireAuth`. The value names a tier, and `reputationGate.tiers` in config sets the score that tier takes, so a deployment can retune it without editing the surface. A tier the running config doesn't define is inert and everyone passes, so an install that doesn't score its accounts never turns traffic away on a score it never computed. `reputationGate.enabled: false` stops every declared gate at once. Denials are a bare 403 `reputation_required` that names neither the score nor the tier. Nothing in the tree declares one yet.
|
||||
- **A specific verified factor:** `requireVerified` (email, and only under `strict_email_verification_required`), `requirePhoneVerified`, `requireCardVerified`. Opt-in, and the factor must actually have been verified, unlike the default-on gate that turns away accounts still carrying a pending verification the abuse harness asked for. `requireAnyVerified: ['phone', 'card']` is the OR of the last two: any listed factor verified at any point passes, as does a paid plan when `card` is listed (a paying account already has a card on file). Otherwise only the factors the deployment can verify are asked for (an SMS provider configured, a card gate an extension reports on); with none verifiable the gate is inert rather than locking the route on a self-hosted install. A denial carries the first verifiable factor's code plus `factors` (the verifiable ones, in the route's order), so a client can lead with one flow and offer the other. `verifiedFactorGate.enabled: false` in config stops every declared gate at once, e.g. during an SMS-provider outage.
|
||||
- **Metered spend:** an endpoint that spends metered resources on the caller's behalf (moving file content, object-store requests, anything else the account is billed for) declares `requireCredits: true`, which turns an account with no budget left away with a 402 before the handler runs. Endpoints that only describe or delete things deliberately don't: an account that has run out still has to see what it has, clear it, and reach its billing pages. Drivers have no route options, so they call `assertActorHasCredits` themselves ([src/backend/services/metering/enforcement.ts](../src/backend/services/metering/enforcement.ts)) — see `KVStoreDriver`, which does it once for every method.
|
||||
|
||||
Provider sign-in requires the account's authenticator or a recovery code when 2FA is enabled, including signup callbacks that resolve an existing account. The five-minute challenge uses the same single-use `/login/otp` and `/login/recovery-code` verification as password sign-in. The GUI carries it in a URL fragment and removes it before initialization. A session and successful popup-return proof are issued only after verification; cancelling retains the popup's signed retry context without marking it authenticated.
|
||||
|
||||
Disabling 2FA through `/user-protected/disable-2fa` requires both password or OIDC revalidation and either the current six-digit authenticator `code` or an unused recovery code in `code`. Recovery codes are consumed atomically. Missing or incorrect codes leave the secret and remaining recovery codes intact. A team's requirement for 2FA still prevents disabling it.
|
||||
|
||||
Email changes through `/user-protected/change-email` require the password or OIDC revalidation cookie, plus a six-digit `code` from the authenticator when 2FA is enabled. Missing or malformed codes return `400 bad_request`; incorrect codes return `401 code_mismatch`. Both include `factor: 'otp'`. Confirmation links are bound to that verified request. A 2FA-enabled account must request a new link if its existing link was issued without second-factor verification.
|
||||
|
||||
### 3. puter.js
|
||||
|
||||
- Add the method to the matching module in [src/puter-js/src/modules/](../src/puter-js/src/modules/), matching the calling conventions of its siblings (promise-returning; positional shortcut plus options form where that's the local pattern).
|
||||
- Validate cheap preconditions client-side and throw `{ message, code }` objects; pass backend errors through unchanged rather than swallowing or re-wrapping them.
|
||||
|
||||
### 4. Types
|
||||
|
||||
Type the method in JSDoc on the implementation, following the [puter.js type rules](../AGENTS.md#types) (overloads, where shapes live, `index.d.ts`), then run `npm run check:puterjs:types`. Declarations are generated from the JSDoc, so there is nothing to keep in sync by hand.
|
||||
|
||||
### 5. Docs
|
||||
|
||||
- Add the method page at [src/docs/src/](../src/docs/src/)`<Area>/<method>.md` — frontmatter (`title`, `description`, `platforms`), syntax, parameters, return value, and at least one runnable example — and update the area overview (`<Area>.md`). Copy the structure of an existing page.
|
||||
- The docs are the contract users code against: signatures, defaults, and return shapes match the implementation exactly.
|
||||
|
||||
### 6. Tests
|
||||
|
||||
- **Backend:** see [backend tests](../AGENTS.md#backend-tests).
|
||||
- **SDK:** add cases to [src/puter-js/tests/api/suites/](../src/puter-js/tests/api/suites/)`<area>.suite.ts` (register new suites in `suites/index.ts`; there is no globbing). One suite runs on node, browser, and workerd via `npm run test:puterjs`; never write per-platform tests. The runners execute the built bundle, so run `npm run build:workerLib` first or the suite silently tests stale code.
|
||||
- **Desktop-rendered UI** (`puter.ui.*`): add a Playwright spec per [src/puter-js/TESTING.md](../src/puter-js/TESTING.md).
|
||||
|
||||
### 7. Security pass
|
||||
|
||||
Run the [security & privacy](../AGENTS.md#security--privacy) check on the diff, and confirm the auth gates are present.
|
||||
|
||||
## Maintaining an existing API
|
||||
|
||||
Changes are **additive by default**:
|
||||
|
||||
- Existing call signatures keep working, including both positional and options-object forms where a method supports them. Say in the PR how existing callers are unaffected.
|
||||
- New parameters are optional, with defaults that reproduce the old behavior exactly.
|
||||
- Never rename, repurpose, or remove existing parameters, response fields, or error codes. Don't change types, ordering guarantees, or which fields are present when.
|
||||
- New behavior that could surprise existing callers goes behind an opt-in flag.
|
||||
- Docs, types, and tests move in the same PR as the behavior; a signature change with stale docs is a bug.
|
||||
- Bug fixes come with a regression test that fails before the fix. Be suspicious of fixes that change observable behavior: someone may depend on the bug. When in doubt, ask a maintainer.
|
||||
|
||||
### Breaking changes
|
||||
|
||||
Rare and deliberate. In order: explicit maintainer sign-off, a documented migration path, and a rollout plan — typically new surface added first, old surface deprecated, old surface removed much later, if ever. Never break as a side effect of a refactor.
|
||||
|
||||
### Deprecating
|
||||
|
||||
The old surface keeps working. Mark it `@deprecated` in the type declarations, note the replacement on its docs page, and stop using it in examples. Removal is a separate, maintainer-approved decision.
|
||||
|
||||
## Definition of done
|
||||
|
||||
- [ ] Backward compatible (or the break was explicitly approved)
|
||||
- [ ] Right home: extension if core never calls it; layered structure either way
|
||||
- [ ] puter.js method matches sibling conventions; errors are `{ message, code }` with stable codes
|
||||
- [ ] Types updated and matching runtime behavior
|
||||
- [ ] Docs page + area overview updated, with a runnable example
|
||||
- [ ] Tests: backend + three-platform SDK suite (+ e2e for desktop-rendered UI)
|
||||
- [ ] Security pass on the diff
|
||||
- [ ] You ran it end to end
|
||||
@@ -1,365 +0,0 @@
|
||||
# Embedded app browser
|
||||
|
||||
`GET /embed/apps` on the deployment's main GUI origin renders the same app browser
|
||||
as the dashboard's Apps tab, without the desktop or dashboard navigation.
|
||||
|
||||
It supports search, paging, saved folders, and saved ordering. It uses the same
|
||||
installed-app and recommendation data as the dashboard, including the user's
|
||||
removed recommendations. Editing, adding, reordering, renaming, and uninstalling
|
||||
are disabled. Selecting an app opens it in a new browser tab at its own URL, the
|
||||
same way the dashboard does, and the host page stays open. The app's token isn't
|
||||
put in the URL: the app's SDK asks the embed for it once the tab loads, so the
|
||||
embed must still be open then. If the iframe is removed first, the app falls back
|
||||
to its normal sign-in. Apps that need the desktop (built-in and godmode apps,
|
||||
private apps the user can't open, or a failed token request) open `/app/<name>`
|
||||
on the GUI origin, cut off from the embed. Website shortcuts open their external
|
||||
destination with no opener.
|
||||
The embed follows the OS's light or dark setting. A host with a fixed theme adds
|
||||
`theme=dark` or `theme=light` to the fragment (`#user=<uuid>&theme=dark`), since
|
||||
Chromium doesn't pass an iframe's `color-scheme` on to the page inside it.
|
||||
Apps opened from the embed carry `utm_source=puter.com&utm_medium=referral&utm_campaign=app_browser_embed`;
|
||||
the desktop and dashboard send `utm_campaign=desktop` and `utm_campaign=dashboard`.
|
||||
|
||||
## Session and framing
|
||||
|
||||
The embed shows the GUI session only to the user the host page is signed in as.
|
||||
The host names that user's `uuid` (from `puter.auth.getUser()`) in the iframe
|
||||
URL's fragment, as `/embed/apps#user=<uuid>`. The embed reads the session, asks
|
||||
`/whoami` which user it belongs to, and shows the signed-out view unless the two
|
||||
match. So a host that names no user, a malformed one, or a user other than the
|
||||
session's, gets the signed-out view, and so does a session the API refuses. A
|
||||
failed `/whoami` check shows the error state. The fragment never reaches the
|
||||
server and can only narrow what the embed shows: naming the session's own user
|
||||
gives the same view as a direct visit to the GUI. Changing the fragment of a
|
||||
loaded embed reloads it.
|
||||
|
||||
The embed reads the existing GUI session from local storage on its own origin.
|
||||
If that storage holds no session, the embed gets the session from the GUI origin's session cookie through
|
||||
`/get-gui-token`. This matters in Safari, which gives a framed origin separate
|
||||
local storage under each top-level origin, so even a sibling subdomain's iframe
|
||||
can't see the GUI's stored token. The session cookie is still sent, because the
|
||||
host is on the same site. The embed doesn't write that token back to storage. It
|
||||
does not create accounts, show a login dialog, or accept a token from its parent.
|
||||
Being signed in on a subdomain alone does not create a session on the main GUI
|
||||
origin.
|
||||
|
||||
Without a session it can show, the embed shows the recommended apps (`RecommendedAppsService`)
|
||||
that the server put in the page, and nothing else: no installed apps, folders, or
|
||||
saved order, and no puter.js. Selecting one opens it at the same URL a signed-in
|
||||
launch uses, with no opener and no token on offer, so the app signs in on its own.
|
||||
Apps that need the desktop open `/app/<name>`. If the page carries no recommended
|
||||
apps, or a load fails, the embed displays an unavailable/error state.
|
||||
|
||||
The route's `Content-Security-Policy: frame-ancestors` allows the GUI origin and
|
||||
the embedding page's origin when it is exactly one label under `config.domain`
|
||||
with the scheme and port from `config.origin`. Subdomains the server answers
|
||||
itself (`api`, `dav`, `js`, `www`, the native-app hosts, and the host of
|
||||
`api_base_url`) and the user-content hosting domains (`static_hosting_domain`,
|
||||
`private_app_hosting_domain`, and their `_alt` variants) are refused. For
|
||||
example, `https://puter.com` permits `https://spreadsheet.puter.com` but not
|
||||
`https://api.puter.com`, and `http://puter.localhost:4100` permits
|
||||
`http://spreadsheet.puter.localhost:4100` but not
|
||||
`http://someone.site.puter.localhost:4100`. The embedding page is read from the
|
||||
iframe request's referrer, so the host must not suppress it (the browser default
|
||||
`strict-origin-when-cross-origin` is enough). All ancestors in a nested iframe
|
||||
chain must be allowed. Other GUI routes retain `X-Frame-Options: SAMEORIGIN`.
|
||||
|
||||
The embed accepts no query parameters. Requests with a query string redirect to
|
||||
the same embed path so SDK bootstrap parameters cannot override the session or API
|
||||
origin. The host needs to allow the GUI origin in its own `frame-src` CSP if it
|
||||
sets one. If sandboxing the iframe, allow scripts, same-origin storage, popups,
|
||||
and popups to escape the sandbox.
|
||||
|
||||
## Host integration
|
||||
|
||||
A first-party app that signs in with puter.js can place `<puter-user-menu>` from
|
||||
the SDK in its header instead (see `src/puter-js/src/ui/components.md`). It opens
|
||||
this embed with the signed-in user named and closes it when the app's sign-in
|
||||
changes. The rest of this section is for a host that builds the iframe itself.
|
||||
|
||||
The iframe can be shown whether or not the user is signed in. Create it when
|
||||
“More apps” is opened, naming the signed-in user in its fragment, and remove it
|
||||
on close so reopening loads current apps and preferences. Remove it when the
|
||||
host's own sign-in changes too, such as on sign-out, so an open embed doesn't
|
||||
keep the previous user. The host owns the modal and close control;
|
||||
there is no parent messaging API and no app list or credential is sent to it.
|
||||
The app browser fills its iframe and responds to its size. It shows a spinner until its apps load.
|
||||
|
||||
This example assumes `puterOrigin` comes from the host app's deployment
|
||||
settings. Translate the button and frame labels using the host app's
|
||||
localization system.
|
||||
|
||||
```html
|
||||
<button id="more-apps" type="button">More apps</button>
|
||||
<dialog id="apps-dialog" aria-label="More apps">
|
||||
<button id="close-apps" type="button">Close</button>
|
||||
<div id="apps-frame"></div>
|
||||
</dialog>
|
||||
<style>
|
||||
#apps-dialog { padding: 12px; width: min(900px, 90vw); }
|
||||
#apps-frame iframe { width: 100%; height: min(650px, 75vh); border: 0; }
|
||||
</style>
|
||||
<script type="module">
|
||||
const puterOrigin = 'https://puter.com'; // Use this deployment's configured origin.
|
||||
const dialog = document.querySelector('#apps-dialog');
|
||||
const container = document.querySelector('#apps-frame');
|
||||
const trigger = document.querySelector('#more-apps');
|
||||
|
||||
trigger.addEventListener('click', async () => {
|
||||
const url = new URL('/embed/apps', puterOrigin);
|
||||
// Signed out, the embed shows the recommended apps.
|
||||
const user = puter.auth.isSignedIn() ? await puter.auth.getUser().catch(() => null) : null;
|
||||
if (user) url.hash = `user=${encodeURIComponent(user.uuid)}`;
|
||||
const frame = document.createElement('iframe');
|
||||
frame.title = 'Your apps';
|
||||
frame.src = url.href;
|
||||
container.replaceChildren(frame);
|
||||
dialog.showModal();
|
||||
});
|
||||
document.querySelector('#close-apps').addEventListener('click', () => dialog.close());
|
||||
dialog.addEventListener('close', () => {
|
||||
container.replaceChildren();
|
||||
trigger.focus();
|
||||
});
|
||||
</script>
|
||||
```
|
||||
|
||||
## Implementation
|
||||
|
||||
`createAppBrowser` in `src/gui/src/UI/Dashboard/AppBrowser.js` owns the shared UI
|
||||
and data behavior; its `guestApps` option is the signed-out list. `TabApps.js`
|
||||
injects desktop actions. `appsEmbed.js` supplies new-tab launching through
|
||||
`launchAppInTab` (signed in) and `guestAppLaunchUrl` (signed out) in
|
||||
`src/gui/src/helpers/appTabLaunch.js`, which build the same app URL as
|
||||
`launch_app`, and loads only the embed entry, its SDK, and shared styles. `embedHostUser` in
|
||||
`src/gui/src/helpers/embedHostUser.js` reads the host's user from the fragment, and
|
||||
`embedHostTheme.js` beside it applies the host's theme.
|
||||
The normal GUI webpack build emits `bundle.min.js`, `apps-embed.min.js`, and `feedback-embed.min.js`.
|
||||
The `/embed/apps` route puts the recommended list in the page's config as
|
||||
`guestApps`. No puter.js API or app-list endpoint changes are needed.
|
||||
|
||||
## Feedback dialog
|
||||
|
||||
`GET /embed/feedback` is the app feedback dialog (`puter.ui.showFeedbackDialog()`)
|
||||
for a first-party page, with the same framing rules, query redirect, and
|
||||
host-user binding as `/embed/apps`. It sends the feedback for the framing
|
||||
page's origin, which the server reads from the referrer and pins with
|
||||
`frame-ancestors`; opened top-level, it does nothing. Its page is transparent
|
||||
so the dialog's own scrim covers the host.
|
||||
|
||||
puter.js uses it on a page one label under the GUI origin, other than the
|
||||
subdomains the server refuses, when the page is signed in, framing it in a
|
||||
modal `<dialog>` that stays hidden until the embed is ready. A session check that takes longer than 200ms shows a loading scrim
|
||||
with a spinner in its place; Escape or a click on it cancels. The embed posts `{ msg: 'feedbackEmbed', state }` to its parent
|
||||
(`loading`, then `ready` or `unavailable`) and finally
|
||||
`{ msg: 'feedbackDialogClosed', sent }`. It reports `unavailable` without the
|
||||
named user's session, or while that user's email still needs confirming, and
|
||||
puter.js then uses the `/action/send-feedback` popup, which can sign in. So does
|
||||
a frame that never answers, such as one the host's CSP blocks. `feedbackEmbed.js`
|
||||
is the entry, sharing `embedSession` in `src/gui/src/helpers/embedSession.js`
|
||||
with the app browser; `showFeedbackEmbed` in `src/puter-js/src/lib/guiEmbed.js`
|
||||
is the host side.
|
||||
|
||||
## Sign-in and file pickers
|
||||
|
||||
`GET /embed/sign-in`, `/embed/show-open-file-picker`, `/embed/show-save-file-picker`
|
||||
and `/embed/show-directory-picker` serve the same GUI popups as `/action/<action>`
|
||||
for a first-party page, with the same framing rules, query redirect, and
|
||||
`no-store` as `/embed/apps`. Any other spelling of the path (another case, a
|
||||
trailing slash) redirects to it, since the shell boots the desktop for it. They render the normal GUI shell, which names the
|
||||
framing page (read from the referrer, pinned with `frame-ancestors`) as
|
||||
`gui_params.embed_host_origin`. The GUI then runs the popup boot unchanged, with
|
||||
the framing page as `window.opener`, and `window.close()` asks the page to
|
||||
remove the frame (`src/gui/src/helpers/popupFrame.js`). Without a framing page
|
||||
to answer (opened top-level, or framed by a refused page) it does nothing. The
|
||||
popup's parameters come from the fragment instead of the query: `msg_id`,
|
||||
`options` and `request_auth`, plus `user` for a picker.
|
||||
|
||||
The frame posts `{ msg: 'puterPopupFrame', state }` to its parent: `loading` as
|
||||
soon as the shell boots, `ready` once it shows a window or dialog, `unavailable`
|
||||
when the page has to use the popup, `provider` when the user picks a sign-in
|
||||
provider (below), and `closed` where a popup would close. Its
|
||||
answers (`puter.token`, `fileOpenPicked`, `directoryPicked`, `fileSaved`, the
|
||||
cancel messages, `sendMeFileData`) are the popup's.
|
||||
|
||||
puter.js uses them on a page one label under the GUI origin, other than the
|
||||
subdomains the server refuses, unless the page is cross-origin isolated or in a
|
||||
frame of another site (which `frame-ancestors` refuses too).
|
||||
`showPopupFrame` in `src/puter-js/src/lib/guiEmbed.js` shows the frame in a box
|
||||
the size of the popup window, over a scrim, hidden until `ready` and behind the
|
||||
loading spinner after 200ms. Escape closes it, on the page or in the frame. One shows per popup, as each
|
||||
popup has its own window name: a second picker request of the same kind
|
||||
replaces the one showing, and a `signIn()` call made while a framed sign-in is
|
||||
open settles with it. On
|
||||
screens up to 600px wide the box fills the screen. A frame that is refused,
|
||||
never answers, or reports `unavailable` falls back to the popup, opened through
|
||||
a consent dialog when the click that asked for it has expired. A reload or navigation the frame
|
||||
starts itself (logging out, a sign-out in another tab) names the frame as its
|
||||
referrer, so the server refuses it; the page then shows its spinner and opens
|
||||
the frame again, once per `ready`.
|
||||
|
||||
- **Sign-in** (`puter.auth.signIn()`) has no host-user binding: it is how the
|
||||
page gets a user. The box has a close button, which rejects with
|
||||
`auth_window_closed` as closing the popup does. Providers refuse to be
|
||||
framed, so choosing one posts `{ state: 'provider', url }` with the
|
||||
provider's start URL for this sign-in's `msg_id`, then `unavailable`. The
|
||||
page opens the popup straight at that URL, once it checks the URL is the
|
||||
GUI's `/auth/oidc/<provider>/start` for this `msg_id`, and the provider
|
||||
returns it as this sign-in's popup. `attempt_temp_user_creation` keeps the popup, since
|
||||
only the popup creates a temporary account.
|
||||
- **Pickers** frame only for a signed-in page. The GUI reports `unavailable`
|
||||
unless its session belongs to the user named in `user`, and the page then uses
|
||||
the popup, which can sign in and switch accounts. The token the GUI sends
|
||||
before a picker's answer isn't adopted, since it is for the page's own user.
|
||||
Dismissing the loading scrim settles the picker as cancelled.
|
||||
|
||||
## Upgrade
|
||||
|
||||
`GET /embed/upgrade` is the deployment's upgrade dialog for a first-party page,
|
||||
served like the popups above: the GUI shell with the same framing rules, query
|
||||
redirect, exact-path redirect and `no-store`, naming the framing page as
|
||||
`gui_params.embed_host_origin`. It runs only when framed by an allowed page.
|
||||
|
||||
puter.js frames it on a signed-in first-party page (the same pages as the
|
||||
pickers) for `puter.ui.requestUpgrade(details)`, and for the Upgrade button of
|
||||
the usage limit dialog a refused call shows there. Elsewhere both keep their
|
||||
behavior: inside an app `requestUpgrade()` asks the desktop, and on any other
|
||||
page the usage limit dialog opens the dashboard's upgrade page in a new tab.
|
||||
|
||||
The frame covers the page with no box or close button, and the GUI page is
|
||||
transparent (`html.popup-frame-transparent`) so the dialog's own scrim covers
|
||||
the host. The frame allows `payment` as well as `otp-credentials`. Escape on the
|
||||
page is left to the dialog, since one there shouldn't drop a payment in
|
||||
progress. Its fragment names `user` (the page's signed-in user) and, when the
|
||||
refusal gave them, `reason` (`funds`, `subscription` or `storage`) and `method`.
|
||||
|
||||
The GUI reports `unavailable` unless its session belongs to `user`, the account
|
||||
has no verification or password change waiting (which the frame couldn't be
|
||||
closed from), and the deployment defines `window.UIUpgradeAccount` with
|
||||
`UIUpgradeAccount.embeddable === true`. Otherwise it runs:
|
||||
|
||||
```js
|
||||
new UIUpgradeAccount({ attribution: { appUid, reason, method }, onClose }).open_as_window();
|
||||
```
|
||||
|
||||
`appUid` is the app the framing page's origin resolves to, never one the
|
||||
fragment names. `open_as_window()` resolves once the dialog shows, and the GUI
|
||||
then posts `ready`. While the page still hides the frame, which Chromium and
|
||||
WebKit lay out at 0x0, the GUI posts `ready` first and opens the dialog once
|
||||
the frame has a viewport, so a window it opens doesn't center on 0x0.
|
||||
`onClose({ upgraded })` must run exactly once, when the
|
||||
dialog closes or when it declines to open for the account; the GUI then posts
|
||||
`{ msg: 'puterPopupFrame', state: 'closed', upgraded }` and the page removes
|
||||
the frame. If `open_as_window()` rejects, the GUI posts `unavailable` instead. `requestUpgrade()` resolves `{ upgraded }`. A call made while the
|
||||
frame shows settles with it rather than replacing it, and a call refused while
|
||||
it shows still rejects but opens no usage limit dialog over it.
|
||||
|
||||
When the frame can't show the upgrade (refused, never ready, `unavailable`, or
|
||||
no user to name), puter.js opens the dashboard's upgrade page in a new tab with
|
||||
what was refused, through a consent dialog when the click has expired or the
|
||||
tab is blocked, and `requestUpgrade()` resolves `{ upgraded: false }`.
|
||||
|
||||
## Top-up
|
||||
|
||||
`GET /embed/top-up` is the deployment's top-up dialog, where a user with a plan
|
||||
buys add-on credit, served and framed like the upgrade: a transparent GUI page
|
||||
covering the host with no box or close button, allowing `payment` and
|
||||
`otp-credentials`, with Escape left to the dialog. Its fragment names `user`.
|
||||
|
||||
puter.js frames it for the **Add credits** item of `<puter-user-menu>`, which
|
||||
shows only when whoami reports `subscribed: true` and no `team` paying for the
|
||||
account. On a page that can't frame the GUI, the item opens the dashboard with
|
||||
`?topup=` in a new tab instead.
|
||||
|
||||
The GUI reports `unavailable` unless its session belongs to `user`, the account
|
||||
has no verification or password change waiting, and the deployment defines
|
||||
`window.UITopUp` with `UITopUp.embeddable === true`. Otherwise it runs:
|
||||
|
||||
```js
|
||||
new UITopUp({ onClose }).open();
|
||||
```
|
||||
|
||||
`open()` resolves once the dialog has loaded the account and shows what it can
|
||||
buy, and `onClose({ toppedUp })` must run exactly once, when the dialog closes
|
||||
or when it declines to open for the account. The GUI then posts `{ msg: 'puterPopupFrame', state: 'closed', toppedUp }`
|
||||
and the page removes the frame. When the frame can't show the top-up, puter.js
|
||||
opens the dashboard's top-up in a new tab, as it does the upgrade.
|
||||
|
||||
While the top-up shows, a call refused for funds still rejects but opens no
|
||||
usage limit dialog over it, and `requestUpgrade()` frames nothing over it: it
|
||||
resolves `{ upgraded: false }` once the top-up closes.
|
||||
|
||||
## Email confirmation
|
||||
|
||||
`GET /embed/request-email-confirmation` walks a first-party page's user through
|
||||
confirming their email, served like the upgrade: the GUI shell with the same
|
||||
framing rules, redirects and `no-store`, run only when framed by an allowed
|
||||
page. puter.js frames it on a signed-in first-party page for
|
||||
`puter.ui.requestEmailConfirmation()`. Inside an app the desktop shows the same
|
||||
windows over the app instead.
|
||||
|
||||
Like the upgrade, the frame covers the page with no box or close button, over a
|
||||
transparent GUI page, and its fragment names `user`. Escape closes it, on the
|
||||
page or on the save-account or code window. On a window, Escape closes that
|
||||
window as its close button does, so a code already sent still answers. The GUI
|
||||
reports `unavailable` unless its session belongs to `user` and the account has
|
||||
no phone, card or password check waiting.
|
||||
An email check is fine: the boot skips its own email gate for this action, so
|
||||
the user gets one code window they can close.
|
||||
|
||||
The GUI then runs `ensureConfirmedEmail` (`src/gui/src/helpers/ensureConfirmedEmail.js`),
|
||||
which the desktop's handler uses too: a temporary account gets the save-account
|
||||
window first, then an unconfirmed email gets the code window, each at most
|
||||
once. When the page's whoami already reads the email as confirmed,
|
||||
`requestEmailConfirmation()` resolves `{ response: true }` without framing
|
||||
anything, and a confirmation in the frame marks that whoami confirmed for the
|
||||
calls after it. An email the GUI reads as confirmed shows nothing. Before
|
||||
opening a window the GUI waits for the frame to have a viewport, as the upgrade
|
||||
does. It then posts
|
||||
`{ msg: 'puterPopupFrame', state: 'closed', confirmed }`, and
|
||||
`requestEmailConfirmation()` resolves `{ response: confirmed }`, the field an
|
||||
app reads from the desktop's answer. A call made while the frame shows settles
|
||||
with it. A frame that is refused, never ready, or `unavailable` resolves
|
||||
`{ response: false }`, since there's no popup to fall back to.
|
||||
|
||||
## Custom domains
|
||||
|
||||
`GET /embed/attach-custom-domain` is the custom domain dialog of
|
||||
`puter.hosting.attach()`, the popup at `/action/attach-custom-domain`, served
|
||||
like the upgrade: the GUI shell with the same framing rules, redirects and
|
||||
`no-store`, run only when framed by an allowed page. puter.js frames it on a
|
||||
signed-in first-party page (the same pages as the pickers).
|
||||
|
||||
The frame covers the page with no box or close button, over a transparent GUI
|
||||
page, so the dialog floats over the host with its own scrim as it does on the
|
||||
desktop. The frame allows `clipboard-write`, for the dialog's copy buttons, as
|
||||
well as `otp-credentials`. Escape on the page is left to the dialog, which
|
||||
doesn't close while a request is in flight. Its fragment names `user` (the
|
||||
page's signed-in user), `subdomain` (the site to attach a domain to) and
|
||||
`msg_id`.
|
||||
|
||||
The GUI reports `unavailable` unless its session belongs to `user` and the
|
||||
account has no verification or password change waiting. It then runs the
|
||||
popup's action unchanged, with the framing page as the requester the dialog
|
||||
names: it posts `ready` once the dialog is on the page, answers
|
||||
`{ msg: 'customDomainAttached', domain, original_msg_id }` once the dialog has
|
||||
closed and its exit has played, and then posts `closed`. `domain` is the domain the user chose, or
|
||||
null. `attach()` resolves it. A frame that closes without an answer, or a
|
||||
loading scrim the user dismisses, resolves null.
|
||||
|
||||
A call made while the frame shows resolves null without replacing it, as a
|
||||
second call does on the desktop. A frame that is refused, never ready, or
|
||||
`unavailable` falls back to the popup, which can sign in, through a consent
|
||||
dialog when the click has expired.
|
||||
|
||||
## Checks
|
||||
|
||||
Run the GUI component/browser tests (with the SDK bundle built and Playwright
|
||||
Chromium installed), and the backend route tests:
|
||||
|
||||
```sh
|
||||
npx vitest run --config src/gui/vitest.config.js src/gui/src/UI/Dashboard/AppBrowser src/feedbackEmbed
|
||||
npx vitest run src/puter-js/src/lib/guiEmbed.test.js src/puter-js/src/modules/UI.showFeedbackDialog.test.js src/puter-js/src/modules/Auth.signInFrame.test.js src/puter-js/src/modules/UI.pickerFrame.test.js src/puter-js/src/modules/UI.requestUpgrade.test.js src/puter-js/src/modules/UI.showTopUp.test.js src/puter-js/src/modules/UI.attachCustomDomainFrame.test.js src/puter-js/src/lib/upgradePrompt.test.js src/puter-js/src/modules/UsageLimitDialog.upgrade.test.js
|
||||
npx vitest run --config src/gui/vitest.config.js src/gui/src/helpers/popupFrame.test.js
|
||||
npx vitest run --config src/backend/vitest.config.ts src/backend/server.test.ts
|
||||
```
|
||||
@@ -104,7 +104,7 @@ Puterは[**puter.com**](https://puter.com)でホストサービスとして利
|
||||
|
||||
メンテナーやコミュニティと以下のチャンネルを通じてつながりましょう:
|
||||
|
||||
- バグ報告や機能リクエストは [新規Issue](https://github.com/HeyPuter/puter/issues/new/choose) まで
|
||||
- バグ報告や機能リクエストがありますか? [issueを開く](https://github.com/HeyPuter/puter/issues/new/choose) してください。
|
||||
- Discord: [discord.com/invite/PQcx7Teh8u](https://discord.com/invite/PQcx7Teh8u)
|
||||
- X (Twitter): [x.com/HeyPuter](https://x.com/HeyPuter)
|
||||
- Reddit: [reddit.com/r/puter/](https://www.reddit.com/r/puter/)
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
# Pagination convention for list APIs
|
||||
|
||||
Every list endpoint follows one wire contract. This document is the source of
|
||||
truth for adding pagination to a new or existing list surface.
|
||||
|
||||
## Wire contract
|
||||
|
||||
Requests accept:
|
||||
|
||||
| Param | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `limit` | number | Maximum items per page. Each endpoint documents its cap and default. |
|
||||
| `cursor` | string \| null | Opaque continuation token. `null` (or any presence of the key) requests the first page. |
|
||||
| `offset` | number | Skip N items. Legacy/discouraged — see below. Cannot be combined with `cursor`. |
|
||||
| `includeTotal` | boolean | Adds `total` to the response. |
|
||||
|
||||
Paginated responses are an envelope:
|
||||
|
||||
```json
|
||||
{ "items": [...], "cursor": "…", "total": 123 }
|
||||
```
|
||||
|
||||
- `cursor` is present only while more pages exist. Clients iterate until it
|
||||
is absent.
|
||||
- `total` is present only when the request set `includeTotal`.
|
||||
- **Pages may be short.** Post-query filtering (TTL expiry, permission
|
||||
checks) can shrink a page below `limit` — or even to zero — while a
|
||||
`cursor` is still returned. Never use `items.length < limit` as an
|
||||
end-of-list signal.
|
||||
|
||||
## Backward compatibility
|
||||
|
||||
Requests without pagination params keep returning the full result in the
|
||||
legacy shape (bare array) forever — old clients never break.
|
||||
|
||||
The envelope trigger depends on the endpoint's history:
|
||||
|
||||
- Endpoints where `limit` pre-dates the convention (readdir, subdomain
|
||||
`select`) return the envelope only when the request contains `cursor`
|
||||
(including `null`) or `includeTotal`; `limit`/`offset`-only requests keep
|
||||
the bare array.
|
||||
- Endpoints where every pagination param is new (kv `list`, workers
|
||||
`getFilePaths`) return the envelope when any pagination param is present.
|
||||
|
||||
## Cursors
|
||||
|
||||
Cursors are opaque base64-encoded JSON, produced and consumed only by the
|
||||
backend (`src/backend/util/pagination.ts`). A cursor over a global sequence
|
||||
(an auto-increment `id`) is sealed with `sealCursor`/`openCursor` instead, so
|
||||
two pages can't be decoded into how fast the table grows; `openCursor` still
|
||||
reads a plain cursor. What a cursor wraps is an implementation detail per
|
||||
store:
|
||||
|
||||
- DynamoDB-backed lists wrap `LastEvaluatedKey`. A kv listing of another
|
||||
namespace (cross-app or through a share handle) resumes after the last entry
|
||||
it returned instead, since the scan can stop on a private one, and its cursor
|
||||
is sealed to that listing and padded to one length.
|
||||
- SQL-backed lists wrap a keyset position — `(sortValue, id)` of the last
|
||||
row — and the query seeks past it (`WHERE (col, id) > (?, ?) ORDER BY col,
|
||||
id`). Any SQL list gaining a cursor must have a deterministic `ORDER BY`
|
||||
ending in a unique tiebreaker (`id`).
|
||||
- The notification catch-up (`GET /events/fetch`) holds two keysets in one
|
||||
sealed cursor: the last `id` it returned, and the last `(replaced_at, id)`
|
||||
among rows up to that id. Each is its own indexed read, merged into one
|
||||
page, so a row replaced after a client read it comes back from that cursor.
|
||||
`from=end` with no cursor answers an empty page whose cursor holds the
|
||||
slice's newest `id` and the slice's latest `(replaced_at, id)` as the
|
||||
replacement mark.
|
||||
|
||||
Cursors that carry a sort also pin it: a request that passes a cursor plus a
|
||||
conflicting sort is rejected with 400.
|
||||
|
||||
## Offset
|
||||
|
||||
SQL-backed endpoints support `offset` natively. DynamoDB-backed endpoints
|
||||
(kv) emulate it by advancing past skipped items with `Select: COUNT` queries
|
||||
— no item data is transferred, but read capacity is still consumed for
|
||||
everything skipped, and the caller is metered for it. **Offset is supported
|
||||
for parity, not recommended** — cost grows linearly with the offset, so it is
|
||||
capped (kv: 5000). Use cursors.
|
||||
|
||||
## Totals (`includeTotal`)
|
||||
|
||||
- SQL: `SELECT COUNT(*)` with the same WHERE clause as the listing.
|
||||
- DynamoDB: a `Select: COUNT` loop over the query (TTL-filtered), metered to
|
||||
the caller. Cost is proportional to the total item count — request the
|
||||
total on the first page only, not every page.
|
||||
- Where visibility is decided per-actor after the query (protected apps in
|
||||
the catalog listing), `total` approximates the visible set: it counts
|
||||
non-protected plus caller-owned rows, and misses rows visible only through
|
||||
explicit permission grants.
|
||||
|
||||
## fetchUntilFull (kv only)
|
||||
|
||||
DynamoDB applies `Limit` before its filter expression, so TTL-filtered pages
|
||||
are structurally short. `fetchUntilFull: true` makes the backend keep
|
||||
fetching (bounded number of continuation queries) until the page holds
|
||||
`limit` items or the keyset is exhausted. Requires `limit`. If the bound is
|
||||
hit, the response simply carries a cursor — still convention-legal.
|
||||
|
||||
## Adding pagination to a new endpoint
|
||||
|
||||
1. Use `encodeCursor`/`decodeCursor`/`normalizeLimit`/`normalizeOffset` from
|
||||
`src/backend/util/pagination.ts` — `sealCursor`/`openCursor` when the
|
||||
cursor wraps a global id.
|
||||
2. Push equality filters into the query so pages and counts operate on the
|
||||
true result set; only genuinely per-actor filtering may remain post-query
|
||||
(short-pages rule covers it).
|
||||
3. Fetch `limit + 1` rows to detect whether a next page exists (SQL), or use
|
||||
`LastEvaluatedKey` (DynamoDB).
|
||||
4. Keep the no-params request returning the legacy full result.
|
||||
@@ -1,80 +0,0 @@
|
||||
# Browser push client
|
||||
|
||||
Puter's desktop shows pushes through `/sw.js`. Core shows Settings → Notifications only when the `pushEnabled` gui param is `true`; core never sets it. The worker is registered only when push is turned on in the browser, and a load removes one left without a subscription. A backend extension that delivers pushes sets it with `services.homepage.setGuiParam('pushEnabled', true)` and can load its own code into the worker with `services.homepage.registerServiceWorkerScript(path)`.
|
||||
|
||||
Core subscribes the browser to a push service only when the server also publishes `pushWebPushKey` (see [Built-in Web Push client](#built-in-web-push-client)), and it has no `/push/*` routes of its own. The device, app, preference and subscription endpoints below come from that extension. On a deployment without one, such as most self-hosted servers, they answer core's plain 404 (`code: 'not_found'`), which `puter.notifications` reports as `push_unavailable` and the Notifications tab shows as unavailable. A device the extension can't find is `device_not_found`.
|
||||
|
||||
## Push payload
|
||||
|
||||
`sw.js` reads the push's JSON body. The message is its `puter` field, or `data.puter` for services that only carry string data, as an object or a JSON string:
|
||||
|
||||
```ts
|
||||
{ uid?: string; title?: string; body?: string; icon?: string; group?: string;
|
||||
target?: { app?: string; path?: string }; data?: unknown; channel?: string;
|
||||
type?: 'challenge'; challenge?: string }
|
||||
```
|
||||
|
||||
`uid` is the notification's uid, a later push with the same `group` replaces the one showing, and `target` and `data` reach the app when the user taps it.
|
||||
|
||||
When the browser replaces the subscription (`pushsubscriptionchange`), `sw.js` keeps the new one, or subscribes again with Puter's stored key when the browser gave none and the event, the receipt or an earlier mark names the subscription that went. It writes `puter:subscription-changed:<scope>` to the `puter-push` IndexedDB as `{ at, oldEndpoint }`, the old endpoint kept from an earlier mark no page handled yet, else taken from the event or from the receipt, forgets the receipt kept for that scope, and posts `{ type: 'puter-push-subscription-changed' }` to every open window on Puter's origin. The worker has no token or session, so an open page runs `helpers/pushSync.js` on that message, after any run already in flight, and a later load does the same through the marker.
|
||||
|
||||
## Receipts
|
||||
|
||||
Before a notification is pushed, the server asks the open clients whether they received it live. It sends `notif.receipt` with `{ uid }` on the user's sockets for the app that sent it, or on the desktop's sockets for Puter's own notifications. The SDK remembers the uids its page received through `onNotification()`, `onPush()` or the desktop's feed, and answers once its page has that notification, or with `{ received: false }` when it never gets it. The server waits up to 5 seconds (`notificationAckTimeoutMs`).
|
||||
|
||||
An answer carries `visibilityState` and `focused` as they were when the page received the notification. Pages never show a browser notification themselves; pushes do.
|
||||
|
||||
- An answer from a page that was visible and focused stops the push (`handled` on `notif.delivery.settled`). An older client that doesn't send `focused` counts while visible.
|
||||
- Otherwise the notification is pushed, to every browser the user subscribed. A website's pushes go only to the browsers that subscribed through its own worker, never to Puter's `/sw.js`. An app inside the desktop pushes through `/sw.js`, which shows the app's name and icon.
|
||||
- A client that can't answer, such as an older puter.js, doesn't hold the push back.
|
||||
|
||||
The ask reaches only the sockets in the asking region. So that other regions hear about a page too, a page that receives a notification live while visible and focused also sends `notif.handled` with `{ uid }` on the socket it came on. Core relays it as the node-local `notif.handled` event with the socket's user and app, at most 60 a minute per user; an extension that delivers pushes decides what to do with it.
|
||||
|
||||
## Registering
|
||||
|
||||
The registry is in [src/gui/src/helpers/pushClient.js](../src/gui/src/helpers/pushClient.js). A service script registers its client like this:
|
||||
|
||||
```js
|
||||
service_script(async (api) => {
|
||||
api.use()['push.registerPushClient']?.(client);
|
||||
});
|
||||
```
|
||||
|
||||
`use(id)` throws when nothing is registered under `id`, so the form above also runs on a Puter version without the registry and does nothing there. Any script that runs after the GUI bundle can call `use()['push.registerPushClient']?.(client)` the same way. Registering again replaces the client. Registering dispatches `push-client-registered` on `window`, and an open Notifications tab redraws.
|
||||
|
||||
With no client registered, the tab says push notifications aren't available in this browser.
|
||||
|
||||
### Built-in Web Push client
|
||||
|
||||
When the server also sets the `pushWebPushKey` gui param (a VAPID public key), core registers its own client ([webPushClient.js](../src/gui/src/helpers/webPushClient.js)) unless an extension registered one first. It uses `puter.notifications.enable()` / `disable()`, the same standard Web Push flow apps use on their own origins:
|
||||
|
||||
1. `GET /push/web/key` for the key, then `pushManager.subscribe()` on the `/sw.js` registration.
|
||||
2. `POST /push/web/subscriptions { subscription }`. A subscription the server already has answers `{ status: 'active' }`. A new one answers `{ status: 'pending' }`, and the server sends it a push with `{ type: 'challenge', challenge }`.
|
||||
3. The worker shows that push and posts `{ type: 'puter-push-challenge', challenge }` to the origin's windows; the page sends it to `POST /push/web/subscriptions/verify`. The device is registered once the page hands the confirmation back.
|
||||
4. `DELETE /push/web/subscriptions { endpoint }` removes it.
|
||||
|
||||
Apps on their own origin do the same through `push-sw.js`, which they import into their own worker. A subscription made with an app token is the app's: its pushes are that app's notifications only.
|
||||
|
||||
## Interface
|
||||
|
||||
```ts
|
||||
interface PushClient {
|
||||
enable(options?: { background?: boolean }): Promise<{ deviceUid: string }>;
|
||||
disable(): Promise<void>;
|
||||
status(options?: { cached?: boolean }): Promise<{
|
||||
permission: 'default' | 'granted' | 'denied';
|
||||
registered: boolean;
|
||||
deviceUid?: string;
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
- `enable()` may ask for notification permission, so core calls it from a click: Turn on in the Notifications tab, Allow on a permission prompt that includes `notifications:push` while this browser is off, and the "Turn on notifications" button of the `/action/enable-push` popup and the matching desktop dialog (`UIEnablePushDialog`). Core also calls it without a click, from `helpers/pushSync.js`, when `Notification.permission` is already `granted`, `status()` reports the browser as not registered, the user has not turned push off in this browser, and an app may push to Puter's devices (`puterDevices` in `GET /push/apps`); the browser does not prompt then. That call passes `{ background: true }`: a browser the user removed in Puter stays removed, and the built-in client rejects with code `push_device_removed`, after which core records the browser as turned off. Before a background call the built-in client asks `POST /push/web/subscriptions/status` about every earlier endpoint it knows: the marker's `oldEndpoint` and the last endpoint the server answered `active` for (`puter.push.last-endpoint:<user uuid>` in `localStorage`). On a 429 it registers nothing and rejects with `too_many_requests`, which the sync reports as `later`, keeping the marker for the next run. When any answer is `removed`, it ends the current subscription in the browser, clears the marker and rejects with `push_device_removed`, so a subscription change or a dropped subscription never brings back a browser removed from another device. A click renews it as before. For a click the built-in client runs `puter.notifications.enable()`, which subscribes with the `/sw.js` registration and registers the browser through the Web Push flow above (`platform: 'webpush'`); in the background it runs the same registration without replacing a removed subscription or opening Puter's sign-in. It then resolves the `uid` of this browser's `webpush` device from `listDevices()` and clears the subscription-changed marker. It rejects when permission is refused or the browser wasn't registered.
|
||||
- `disable()` runs `puter.notifications.disable()`, which sends `DELETE /push/web/subscriptions { endpoint }` and ends the browser's subscription. The tab calls it for Turn off, and for Remove on the device marked as this browser. Remove on any other device is `DELETE /push/devices/:uid`.
|
||||
- `status()` reports the state without prompting. `permission` is `Notification.permission`. `registered` is true when this browser has a device registered for the signed-in user and `POST /push/web/subscriptions/status` answers `active` for the current subscription's endpoint, and `deviceUid` is that device's `uid`. A server that can't answer leaves the device match alone, and so does an `unknown` for an endpoint `enable()` registered in this page in the last 5 minutes, since that check can trail a registration. While the subscription-changed marker is set, `registered` is false. With `{ cached: true }` a client may answer with the last registered state it read in this page. Core passes it for the silent registration on app launch and for an app's `puter.notifications.status()`; the Notifications tab and the push dialogs read fresh. The built-in client keeps that state in memory for the same user and subscription for up to 5 minutes, from `enable()` or once the server answered `active` for the endpoint, and answers from it without either request. Reads of the same user and endpoint share one real check while it is in flight. It drops it on `disable()`, a failed `enable()`, an unregistered answer, a set subscription-changed marker, and the worker's `puter-push-subscription-changed` message, so a new page load always asks the server. Whether the calling app may push is checked on every call, against a `GET /push/apps` answer the page reuses for 30 seconds and drops on `enable()`, `disable()`, a grant, a revoke and a user change. When it can't be read, or leaves the app out, the grant alone counts.
|
||||
|
||||
After `enable()` resolves, the tab calls `POST /push/devices/:uid/test` and asks the user whether a notification showed up.
|
||||
|
||||
The tab also reads `GET /push/apps` and `GET`/`POST /push/preferences`.
|
||||
|
||||
Core keeps its own record of a browser the user turned off, `puter.push.browser-off:<user uuid>` in `localStorage`, written by Turn off, by Remove on this browser's device, and by a silent registration the server refuses with `push_device_removed`, and cleared when `enable()` resolves. The silent registration above skips a browser with that record, so a device removed here or from another device stays removed until the user turns it on from a click, while a new sign-in in a browser that already allows notifications registers again.
|
||||
+31
-310
@@ -1,6 +1,6 @@
|
||||
# Self-Hosting Puter
|
||||
|
||||
`docker-compose.yml` brings up Puter **plus every external service it needs** — MariaDB, Valkey, DynamoDB-local, RustFS S3, Caddy — wired together. Closest thing to a production deployment you can self-manage on a single host.
|
||||
`docker-compose.yml` brings up Puter **plus every external service it needs** — MariaDB, Valkey, DynamoDB-local, RustFS S3, nginx — wired together. Closest thing to a production deployment you can self-manage on a single host.
|
||||
|
||||
## One-line installer (recommended)
|
||||
|
||||
@@ -8,21 +8,11 @@
|
||||
curl -fsSL https://raw.githubusercontent.com/HeyPuter/puter/main/install.sh | sh
|
||||
```
|
||||
|
||||
Generates secrets, writes `.env` + `puter/config/config.json`, downloads `docker-compose.yml` + `caddy/Caddyfile` from the OSS repo, and runs `docker compose up -d`. Re-running is safe — it won't overwrite existing config (set `PUTER_FORCE=1` to rotate). Use the manual steps below if you want to inspect or tweak each step yourself.
|
||||
Generates secrets, writes `.env` + `puter/config/config.json`, downloads `docker-compose.yml` from the OSS repo, and runs `docker compose up -d`. Re-running is safe — it won't overwrite existing config (set `PUTER_FORCE=1` to rotate). Use the manual steps below if you want to inspect or tweak each step yourself.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Docker** with the `compose` plugin. This is the one thing you have to install
|
||||
yourself — the installer checks for it and stops with an explanation rather
|
||||
than installing it for you.
|
||||
- **macOS / Windows:** [Docker Desktop](https://docs.docker.com/get-docker/).
|
||||
Both ship the `compose` plugin.
|
||||
- **Linux:** [Docker Engine](https://docs.docker.com/engine/install/) plus the
|
||||
[`docker-compose-plugin`](https://docs.docker.com/compose/install/) package.
|
||||
- Confirm before running anything else: `docker compose version` should print a
|
||||
version, not `command not found`.
|
||||
- If Docker is installed but the shell can't see it, make sure your user is in
|
||||
the `docker` group (Linux) or that Docker Desktop is actually running.
|
||||
- **Docker** with the `compose` plugin.
|
||||
- A **domain** with DNS access — you need a wildcard record (`*.your-domain.com` → server IP). Puter routes by subdomain (`api.<domain>`, `site.<domain>`, `app.<domain>`).
|
||||
- Optional: **TLS certs** (or `certbot` to grab them — see Step 3).
|
||||
|
||||
@@ -30,7 +20,7 @@ Generates secrets, writes `.env` + `puter/config/config.json`, downloads `docker
|
||||
|
||||
| Container | Image | Role |
|
||||
| --------------- | ------------------------ | ---------------------------------------------------------- |
|
||||
| `puter-caddy` | `caddy:2.11-alpine` | Reverse proxy on 80 (and 443 if TLS); forwards to Puter |
|
||||
| `puter-nginx` | `nginx:1.27-alpine` | Reverse proxy on 80 (and 443 if TLS); forwards to Puter |
|
||||
| `puter` | `ghcr.io/heyputer/puter` | The app |
|
||||
| `puter-mariadb` | `mariadb:11` | SQL database — schema applied automatically on first boot |
|
||||
| `puter-valkey` | `valkey/valkey:8-alpine` | Redis-compatible cache + rate-limiter |
|
||||
@@ -45,12 +35,6 @@ Optional services (compose profile `ai`, opt-in):
|
||||
| `puter-ollama` | `ollama/ollama` | Local LLM provider (CPU; GPU passthrough opt-in) |
|
||||
| `puter-ollama-init` | `ollama/ollama` | One-shot — pulls the default model (`tinyllama`) on first boot |
|
||||
|
||||
Optional services (compose profile `smtp`, opt-in):
|
||||
|
||||
| Container | Image | Role |
|
||||
| ------------ | ------------------------ | -------------------------------------------------------- |
|
||||
| `puter-smtp` | `ghcr.io/heyputer/puter` | Receives mail over SMTP and files it into user mailboxes |
|
||||
|
||||
State lives under `./puter/data/<service>/`.
|
||||
|
||||
---
|
||||
@@ -63,7 +47,7 @@ State lives under `./puter/data/<service>/`.
|
||||
MARIADB_ROOT_PASSWORD=$(openssl rand -hex 32)
|
||||
MARIADB_PASSWORD=$(openssl rand -hex 32)
|
||||
S3_SECRET_KEY=$(openssl rand -hex 32)
|
||||
JWT_SECRET_V2=$(openssl rand -hex 64)
|
||||
JWT_SECRET=$(openssl rand -hex 64)
|
||||
URL_SIGNATURE_SECRET=$(openssl rand -hex 64)
|
||||
|
||||
cat > .env <<EOF
|
||||
@@ -80,7 +64,7 @@ S3_SECRET_KEY=$S3_SECRET_KEY
|
||||
S3_BUCKET=puter-local
|
||||
EOF
|
||||
|
||||
mkdir -p puter/config puter/data puter/data/caddy puter/tls
|
||||
mkdir -p puter/config puter/data puter/tls
|
||||
cat > puter/config/config.json <<EOF
|
||||
{
|
||||
"domain": "puter.localhost",
|
||||
@@ -93,7 +77,7 @@ cat > puter/config/config.json <<EOF
|
||||
"private_app_hosting_domain": "app.puter.localhost",
|
||||
"private_app_hosting_domain_alt": "dev.puter.localhost",
|
||||
|
||||
"jwt_secret_v2": "$JWT_SECRET_V2",
|
||||
"jwt_secret": "$JWT_SECRET",
|
||||
"url_signature_secret": "$URL_SIGNATURE_SECRET",
|
||||
|
||||
"database": {
|
||||
@@ -138,10 +122,6 @@ cat > puter/config/config.json <<EOF
|
||||
"ollama": { "enabled": false }
|
||||
},
|
||||
|
||||
"meteringEnforcement": {
|
||||
"subscriptions": false
|
||||
},
|
||||
|
||||
"trust_proxy": 1
|
||||
}
|
||||
EOF
|
||||
@@ -151,32 +131,20 @@ Replace `puter.localhost`, `site.puter.localhost`, `host.puter.localhost`, `dev.
|
||||
|
||||
Why these knobs:
|
||||
|
||||
- `jwt_secret_v2` — the HMAC secret Puter signs and verifies auth tokens with (`kid: 'v2'` JWT header). The pre-v2 token format is retired: a token signed with the old `jwt_secret` no longer verifies, and holders are asked to sign in again. If you are upgrading from a release that had `jwt_secret`, drop it from your config — it is ignored.
|
||||
- `env: "prod"` — the bundled `config.default.json` ships with `env: "dev"` (matches the source-tree `npm run start:gui` workflow, which expects webpack-dev-server emitting a CSS manifest). Self-host runs against pre-built static bundles, so `env: "prod"` makes the homepage emit the `/dist/bundle.min.css` `<link>` tag instead of waiting on a manifest that doesn't exist.
|
||||
- `database.migrationPaths` — Puter applies the bundled MySQL/MariaDB schema on boot. The migration files are idempotent, so it is safe to leave this configured across restarts.
|
||||
- `env: "prod"` — the bundled `config.default.json` ships with `env: "dev"` (matches the source-tree `npm run start=gui` workflow, which expects webpack-dev-server emitting a CSS manifest). Self-host runs against pre-built static bundles, so `env: "prod"` makes the homepage emit the `/dist/bundle.min.css` `<link>` tag instead of waiting on a manifest that doesn't exist.
|
||||
- `database.migrationPaths` — Puter applies the bundled MySQL schema on boot. `mysql_mig_1.sql` (tables) and `mysql_mig_2.sql` (default apps: editor, viewer, pdf, camera, player, recorder, git, dev-center, puter-linux). Idempotent — safe to re-run.
|
||||
- `dynamo.bootstrapTables: true` — Puter creates its KV table on boot. **Only set against a local emulator**, never real AWS.
|
||||
- `dynamo.aws` keys are dummies; DynamoDB-local doesn't validate them but the AWS SDK requires _something_. **Note:** DynamoDB uses `access_key` / `secret_key` (snake_case); S3 below uses `accessKeyId` / `secretAccessKey` (camelCase). Not interchangeable.
|
||||
- `meteringEnforcement.subscriptions: false` — a few surfaces (the OpenAI/Anthropic-compatible AI endpoints) are declared paid-plan-only. A self-hosted install has no paid plans, so every account sits on the free policy and those routes would answer `402 subscription_required` for everyone. Turning the plan gates off opens them; nothing else about metering changes.
|
||||
- `providers.ollama.enabled: false` — Puter auto-probes a local Ollama at `127.0.0.1:11434` by default; without one running you'd see `ECONNREFUSED` on every boot. To run a bundled Ollama, see [Optional: local LLM (Ollama)](#optional-local-llm-ollama) below.
|
||||
- `s3.s3Config.forcePathStyle: true` — RustFS / MinIO / fauxqs need path-style URLs (`<endpoint>/<bucket>`). Real AWS S3 wants virtual-hosted (`<bucket>.<endpoint>`) — drop this flag (or set `false`) when you swap to real S3.
|
||||
- `s3.s3Config.publicEndpoint` — `endpoint` (`http://s3:9000`) only resolves inside the docker network; presigned upload/download URLs handed to the browser need a host-reachable URL. Caddy routes the `s3.<domain>` subdomain to RustFS internally and preserves the Host header end-to-end (required for S3 signature validation), so the browser hits the same port/protocol as the rest of the app — no separate published port, no mixed-content surprises when you turn on TLS. Switch to `https://s3.<your-domain>` once you enable TLS in Step 3. Real AWS S3 doesn't need this — its endpoint is already public; drop the field entirely.
|
||||
- **Bucket CORS** — the browser uploads file bytes straight to RustFS with a cross-origin `PUT` to the presigned URL (Dev Center deploy, `puter.fs.upload`). That only works if the bucket answers the `OPTIONS` preflight with `Access-Control-Allow-*` headers. The `s3-init` container applies those rules on every boot (`put-bucket-cors`, origins `*` — the signed URL is the credential, and the SDK sends no cookies). If deploys fail with `No 'Access-Control-Allow-Origin' header is present`, re-run it: `docker compose run --rm s3-init`. Verify with:
|
||||
|
||||
```bash
|
||||
curl -sk -X OPTIONS "https://s3.<domain>/<bucket>/test" \
|
||||
-H "Origin: https://<domain>" \
|
||||
-H "Access-Control-Request-Method: PUT" \
|
||||
-H "Access-Control-Request-Headers: content-type" -D - -o /dev/null | grep -i access-control
|
||||
```
|
||||
|
||||
You should see `access-control-allow-origin`, `allow-methods: GET, HEAD, PUT, POST, DELETE`, `allow-headers: *`, and `max-age: 3600`.
|
||||
- `trust_proxy: 1` — Caddy terminates TLS and forwards `X-Forwarded-For`. Without this, `req.ip` is the docker-network address of the Caddy container instead of the real client IP, which breaks rate limiting and IP-based audit logs. `1` = one trusted hop (Caddy). Bump to `2` if you put Cloudflare in front of Caddy; never set `true` (it trusts every hop and makes XFF forgeable).
|
||||
- `s3.s3Config.publicEndpoint` — `endpoint` (`http://s3:9000`) only resolves inside the docker network; presigned upload/download URLs handed to the browser need a host-reachable URL. nginx routes the `s3.<domain>` subdomain to RustFS internally and preserves the Host header end-to-end (required for S3 signature validation), so the browser hits the same port/protocol as the rest of the app — no separate published port, no mixed-content surprises when you turn on TLS. Switch to `https://s3.<your-domain>` once you enable TLS in Step 3. Real AWS S3 doesn't need this — its endpoint is already public; drop the field entirely.
|
||||
- `trust_proxy: 1` — nginx terminates TLS and forwards `X-Forwarded-For`. Without this, `req.ip` is the docker-network address of the nginx container instead of the real client IP, which breaks rate limiting and IP-based audit logs. `1` = one trusted hop (nginx). Bump to `2` if you put Cloudflare in front of nginx; never set `true` (it trusts every hop and makes XFF forgeable).
|
||||
|
||||
> If you ever change `MARIADB_PASSWORD` after first boot, `.env` alone won't update MariaDB — its credentials are baked into `./puter/data/mariadb/` on first init. Either rotate the password inside MariaDB by hand or `docker compose down && rm -rf ./puter/data/mariadb` to start fresh.
|
||||
|
||||
## Step 2 — Point DNS at the server \[Optional\]
|
||||
|
||||
In your DNS provider, add records for the main domain plus the subdomains Puter and Caddy route on (`api.*`, `site.*`, `app.*`, `s3.*`):
|
||||
In your DNS provider, add records for the main domain plus the subdomains Puter and nginx route on (`api.*`, `site.*`, `app.*`, `s3.*`):
|
||||
|
||||
```
|
||||
A puter.localhost → <your server's public IP>
|
||||
@@ -191,7 +159,7 @@ A dev.puter.localhost → <your server's public IP>
|
||||
A *.dev.puter.localhost → <your server's public IP>
|
||||
```
|
||||
|
||||
The wildcards are required — Puter routes via subdomains (`api.*`, `app.*`, etc.) and Caddy routes browser S3 traffic via `s3.*` to RustFS.
|
||||
The wildcards are required — Puter routes via subdomains (`api.*`, `app.*`, etc.) and nginx routes browser S3 traffic via `s3.*` to RustFS.
|
||||
|
||||
## Step 3 — TLS (recommended for public installs) \[Optional\]
|
||||
|
||||
@@ -210,15 +178,13 @@ sudo certbot certonly --manual --preferred-challenges dns \
|
||||
|
||||
The cert needs to cover `*.puter.localhost` so that `s3.puter.localhost` (browser S3 endpoint), plus Puter's own `api.*` / `app.*` subdomains, all validate.
|
||||
|
||||
> **Why not Caddy's automatic HTTPS?** Caddy issues certs over ACME by itself, but only for hostnames it knows up front, and only over HTTP-01 with the stock image. Puter serves every user's site and app on a subdomain it invents at runtime (`<name>.site.<domain>`, `<name>.app.<domain>`), which needs a **wildcard** cert — and wildcards require DNS-01, which requires a DNS-provider plugin that isn't in `caddy:2.11-alpine`. So `caddy/Caddyfile` sets `auto_https off` and reads the cert you supply below. If you'd rather have Caddy manage certs, build an image with the plugin for your DNS provider (`xcaddy build --with github.com/caddy-dns/<provider>`) and swap the `tls` line for a `tls { dns <provider> … }` block.
|
||||
|
||||
Drop the resulting `fullchain.pem` and `privkey.pem` into `./puter/tls/`.
|
||||
|
||||
**Wire Caddy to use them:**
|
||||
**Wire nginx to use them:**
|
||||
|
||||
1. Open [caddy/Caddyfile](../caddy/Caddyfile) and uncomment the `# :443 { … }` block at the bottom.
|
||||
2. (Optional but recommended) Replace the plain `:80 { import puter_routes }` block with the `redir` version shown alongside it, to force HTTPS everywhere.
|
||||
3. In [docker-compose.yml](../docker-compose.yml), uncomment the `443:443` port mapping under the `caddy` service.
|
||||
1. Open [nginx/nginx.conf](../nginx/nginx.conf), uncomment **both** `# server { listen 443 ssl … }` blocks (one for `s3.*`, one for the catch-all).
|
||||
2. (Optional) Replace the body of the port-80 blocks with `return 301 https://$host$request_uri;` to force HTTPS everywhere.
|
||||
3. In [docker-compose.yml](../docker-compose.yml), uncomment the `443:443` port mapping under the `nginx` service.
|
||||
4. In `.env`, uncomment `HTTPS_PORT=443`.
|
||||
5. In `config.json`, switch:
|
||||
```json
|
||||
@@ -226,39 +192,11 @@ Drop the resulting `fullchain.pem` and `privkey.pem` into `./puter/tls/`.
|
||||
```
|
||||
…and update the S3 public endpoint:
|
||||
```json
|
||||
"s3": { "s3Config": { "publicEndpoint": "https://s3.puter.localhost", ... } }
|
||||
"s3": { "s3Config": { "publicEndpoint": "https://s3.puter.local", ... } }
|
||||
```
|
||||
|
||||
## Running behind your own reverse proxy
|
||||
|
||||
The bundled `puter-caddy` mirrors production and is the supported default — it terminates TLS (Step 3) and forwards every Host to Puter. But plenty of self-hosters already run their own edge proxy (Traefik, nginx, HAProxy, a cloud load balancer, another Caddy). You can put Puter behind it instead; Puter doesn't terminate TLS itself in any setup, so "your proxy does TLS" is just a matter of pointing it at the Puter container and getting a handful of details right.
|
||||
|
||||
You can keep the bundled Caddy as the single hop your proxy talks to, or bypass it and forward straight to the Puter container on port `4100` (uncomment the `4100:4100` mapping under the `puter` service in [docker-compose.yml](../docker-compose.yml), or attach your proxy to the compose network). Either works — what matters is the rules below.
|
||||
|
||||
**The rules that actually matter** (getting any of these wrong is what causes the redirect loops and "Invalid Host header" failures people hit):
|
||||
|
||||
1. **Don't rewrite the `Host` header.** Puter routes entirely on Host — `api.<domain>`, `site.<domain>`, `app.<domain>` are all distinguished by the incoming host. Forward the original host through unchanged. Do **not** rewrite external hostnames to an internal name like `puter.local`; that forces you to remap every subdomain by hand and still breaks signed-URL and CORS checks. Most proxies preserve Host by default — just don't override it.
|
||||
|
||||
2. **The external domain must equal `domain` in your `config.json`.** If users reach the box at `puter.example.com`, then `domain` must be `puter.example.com` and the hosting domains must be its real subdomains (`site.puter.example.com`, `app.puter.example.com`, …) — exactly as the installer/`config.json` lays them out. Puter rejects hosts it doesn't recognize with `Invalid Host header`.
|
||||
|
||||
3. **Set `protocol` to match the public scheme.** When your proxy terminates TLS, set `"protocol": "https"` (installer: `PUTER_PROTOCOL=https`). Puter builds its origins, redirects, signed S3 URLs, and OIDC callback URLs from this — leave it `http` behind an HTTPS proxy and you get redirect loops, broken logins, and mixed-content errors.
|
||||
|
||||
4. **Forward the standard proxy headers.** Puter needs:
|
||||
- `Host` — the original request host (rule 1).
|
||||
- `X-Forwarded-Proto` — the **external** scheme (`https`), so Puter knows TLS was terminated upstream.
|
||||
- `X-Forwarded-For` — the real client IP (rate limiting + audit logs).
|
||||
- `Upgrade` / `Connection` — passed through for WebSocket / socket.io upgrades, or the realtime connection silently fails.
|
||||
|
||||
5. **Set `trust_proxy` to the number of hops in front of Puter.** One external proxy talking directly to Puter → `"trust_proxy": 1` (installer: `PUTER_TRUST_PROXY=1`). A second proxy in front (e.g. Cloudflare → your proxy → Puter) → `2`. This is the count of proxies between the client and Puter, **including** the bundled Caddy if you keep it. Too low and `req.ip` becomes a proxy address (breaks rate limiting); never set `true` (it trusts every hop and makes `X-Forwarded-For` forgeable).
|
||||
|
||||
6. **Route the wildcard to Puter.** Your proxy must forward `*.<domain>` (covering `api.*`, `app.*`, and `s3.<domain>`) and `*.site.<domain>` (and any other hosting domains) to Puter — same wildcard DNS as Step 2. Puter does the per-subdomain routing internally; the proxy just needs to hand it the traffic with the Host intact.
|
||||
|
||||
With those in place the rest of the stack is unchanged — `docker compose up -d` as below.
|
||||
|
||||
## Step 4 — Bring it up
|
||||
|
||||
Both [docker-compose.yml](../docker-compose.yml) and [caddy/Caddyfile](../caddy/Caddyfile) need to sit next to your `.env` — clone the repo, or copy those two paths out of it. The Caddyfile is bind-mounted read-only; if it's missing, Docker creates a *directory* at that path and `puter-caddy` dies with "not a directory".
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
@@ -273,15 +211,15 @@ Healthy startup:
|
||||
|
||||
```
|
||||
[config] override from /etc/puter/config.json
|
||||
[mysql] running migrations from /opt/puter/dist/src/backend/clients/database/migrations/mysql: <n> file(s)
|
||||
[mysql] running migrations from /opt/puter/dist/src/backend/clients/database/migrations/mysql: 2 file(s)
|
||||
[mysql] applied mysql_mig_1.sql (...)
|
||||
...
|
||||
[mysql] applied mysql_mig_2.sql (9 statements)
|
||||
```
|
||||
|
||||
Then open **<https://puter.localhost>** (or `http://` if you skipped TLS). Login is `admin`; its temporary password is printed in the puter container logs on every boot until you change it:
|
||||
Then open **<https://puter.local>** (or `http://` if you skipped TLS). Login is `admin` — the temp password is printed once in the puter container logs on first boot:
|
||||
|
||||
```bash
|
||||
docker compose logs puter | grep "password for admin"
|
||||
docker compose logs puter | grep tmp_password
|
||||
```
|
||||
|
||||
Change it in Settings after first login.
|
||||
@@ -290,26 +228,6 @@ Change it in Settings after first login.
|
||||
|
||||
All optional. Drop any of the blocks below into `puter/config/config.json` and `docker compose restart puter`. See [config.template.jsonc](../config.template.jsonc) for the full list. Per-key documentation lives in [src/backend/types.ts](../src/backend/types.ts).
|
||||
|
||||
### PostgreSQL database
|
||||
|
||||
> **Community contribution — expect rough edges.** PostgreSQL support was contributed by the community and is not run by Puter.com production. It boots, applies the bundled schema, and exercises the common user/app/fsentry/session/permission/OIDC flows in the integration tests, but less-traveled SQL call sites may still need porting. If you hit a query that doesn't work on Postgres, please open an issue. For production self-hosting today, MariaDB/MySQL and SQLite are the supported defaults.
|
||||
|
||||
The bundled Docker Compose stack still defaults to MariaDB. To use PostgreSQL instead, run a PostgreSQL service yourself, point Puter at it, and use the PostgreSQL migration path:
|
||||
|
||||
```json
|
||||
"database": {
|
||||
"engine": "postgres",
|
||||
"host": "postgres",
|
||||
"port": 5432,
|
||||
"user": "puter",
|
||||
"password": "...",
|
||||
"database": "puter",
|
||||
"migrationPaths": ["/opt/puter/dist/src/backend/clients/database/migrations/postgres"]
|
||||
}
|
||||
```
|
||||
|
||||
You may use `"connectionString": "postgres://puter:...@postgres:5432/puter"` instead of the host/user/password fields. Puter only needs normal PostgreSQL connection details and the migration path; CloudNativePG is one possible Kubernetes operator, but no operator-specific manifests are required by Puter.
|
||||
|
||||
### Email (SMTP)
|
||||
|
||||
Used for password resets, email confirmation, and notifications. Without it those flows silently fail.
|
||||
@@ -326,32 +244,6 @@ Used for password resets, email confirmation, and notifications. Without it thos
|
||||
|
||||
To require email confirmation before login, also set `"strict_email_verification_required": true`.
|
||||
|
||||
Account and notification mail is sent on a named stream: `auth` (password, address and 2FA changes), `signup` (address confirmation; a `puter.signup.validate` listener may name another: lowercase letters, digits, `:`, `_` and `-`, up to 64 characters, starting with a letter or digit) and `notify` (share, app feedback and contact form mail). Add `"streams"` inside `"email"` to configure one, for instance to let your mail provider route or report each kind of mail separately:
|
||||
|
||||
```json
|
||||
"streams": {
|
||||
"notify": {
|
||||
"from": "\"Puter\" <[email protected]>",
|
||||
"headers": { "X-Example-Stream": "notify" },
|
||||
"tagsHeader": "X-Example-Tags"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`headers` are added to every message on the stream, `from` replaces the default sender, and `tagsHeader` names the header that carries a message's tags as `name=value` pairs joined by `, `. A stream that is not listed adds nothing: the message goes out with the default from address and the caller's headers. `streams` is never handed to the transport.
|
||||
|
||||
To read the mail yourself instead of sending it, point the transport at a local SMTP sink — [MailHog](https://github.com/mailhog/MailHog) catches everything and serves it at `http://localhost:8025`. Publish both ports; a container that only exposes them is unreachable from the host:
|
||||
|
||||
```sh
|
||||
docker run -d -p 1025:1025 -p 8025:8025 mailhog/mailhog
|
||||
```
|
||||
|
||||
```json
|
||||
"email": { "from": "\"Puter\" <[email protected]>", "host": "127.0.0.1", "port": 1025, "secure": false, "ignoreTLS": true }
|
||||
```
|
||||
|
||||
Share email is on once a transport exists: a recipient who already has an account is emailed as well as notified in-app, and they opt out with the unsubscribe link the mail carries. Set `"share_email_notifications": false` to keep those in the app only — an address with no account is still emailed, since there is nothing else to reach them with.
|
||||
|
||||
### Sign in with Google (or another OIDC provider)
|
||||
|
||||
```json
|
||||
@@ -418,56 +310,14 @@ Full provider list (chat, image, video, TTS, OCR) is in the template.
|
||||
|
||||
### Per-user storage quota
|
||||
|
||||
Storage is unlimited by default (bounded by host disk). To cap each user:
|
||||
Default is 100 MB per user.
|
||||
|
||||
```json
|
||||
"storage_capacity": 5368709120, // 5 GB
|
||||
"is_storage_limited": true
|
||||
```
|
||||
|
||||
`storage_capacity` defaults to 100 MB.
|
||||
|
||||
### Usage metering and budgets
|
||||
|
||||
Puter meters what an account costs to serve — bytes sent back, object-store
|
||||
requests, KV capacity, AI tokens — against a monthly budget, and refuses the
|
||||
operations that spend it once that budget is gone. The refusal is a `402` with
|
||||
code `insufficient_funds`; reads that only describe things, and every kind of
|
||||
deletion, stay available so an account can always see what it has and clear it.
|
||||
|
||||
On a self-hosted install this is almost certainly not what you want. There is
|
||||
nowhere to buy more, so accounts are held to the free monthly allowance
|
||||
(US$0.50 of measured cost for a registered account, roughly 2.5 GiB of
|
||||
downloads) and start being refused after that. Turn it off:
|
||||
|
||||
```json
|
||||
"unlimitedMetering": true
|
||||
```
|
||||
|
||||
Every account then resolves to an unlimited policy. Usage is still recorded, so
|
||||
the dashboard still shows what is being consumed; nothing is ever refused for
|
||||
lack of budget.
|
||||
|
||||
To keep the budgets but stop them blocking anything — recording only:
|
||||
|
||||
```json
|
||||
"meteringEnforcement": { "enabled": false }
|
||||
```
|
||||
|
||||
Calls driven by a deployed worker are exempt from enforcement by default,
|
||||
because a worker has no prompt to show and nobody watching it fail. Set
|
||||
`"meteringEnforcement": { "workers": true }` to include them.
|
||||
|
||||
Monthly usage records expire on their own once they're old enough to be of no
|
||||
further use — `meteringRetentionMonths` full months past the month they
|
||||
belong to, 3 by default. Set it to `0` to keep every month's records forever:
|
||||
|
||||
```json
|
||||
"meteringRetentionMonths": 6
|
||||
```
|
||||
|
||||
Changing it, including to `0`, applies to records first written after the
|
||||
change; records already stamped keep their expiry.
|
||||
Set `is_storage_limited: false` for unlimited (bounded by host disk).
|
||||
|
||||
### Captcha on signup / login
|
||||
|
||||
@@ -479,19 +329,9 @@ Built-in proof-of-work captcha — no external service needed.
|
||||
|
||||
`difficulty` is one of `easy` / `medium` / `hard`.
|
||||
|
||||
### Disable new signups
|
||||
### Block disposable email TLDs
|
||||
|
||||
Force visitors to log in with an existing account instead of creating a
|
||||
temporary or permanent one.
|
||||
|
||||
```json
|
||||
"disable_user_signup": true
|
||||
```
|
||||
|
||||
### Block disposable email domains
|
||||
|
||||
Addresses on these domains (suffix match, so subdomains too) can't sign up or be
|
||||
set as an account's email.
|
||||
Only enforced when `env: "prod"`.
|
||||
|
||||
```json
|
||||
"blockedEmailDomains": ["mailinator.com", "tempmail.com", "guerrillamail.com"]
|
||||
@@ -538,125 +378,14 @@ Without `--profile ai`, the `ollama` containers stay down and Puter (with `enabl
|
||||
|
||||
For GPU acceleration (NVIDIA), uncomment the `deploy:` block under the `ollama` service in [docker-compose.yml](../docker-compose.yml). Requires `nvidia-container-toolkit` on the host.
|
||||
|
||||
## Optional: receiving mail over SMTP
|
||||
|
||||
Puter can give each account a mailbox at a domain you control. Two pieces are
|
||||
involved, and they are configured together under `userEmail`:
|
||||
|
||||
- an **ingress endpoint** (`POST /email/ingress`) that files a message into the
|
||||
recipient's `~/.mail`. It exists whenever `userEmail.secret` is set.
|
||||
- the **`puter-smtp` receiver**, which accepts mail from other mail servers and
|
||||
hands each message to that endpoint. It runs as its own process, behind the
|
||||
`smtp` compose profile.
|
||||
|
||||
> **Read this before you expose it.** An inbound mail server is unauthenticated
|
||||
> by nature: anyone on the internet may hand it a message for any address in
|
||||
> `localDomains`. Puter performs **no** spam filtering and **no** SPF, DKIM or
|
||||
> DMARC checks — whatever is delivered is stored in the recipient's mailbox and
|
||||
> counts against their storage. If that is not acceptable, put a filtering relay
|
||||
> in front of it and point `localDomains` at that instead.
|
||||
|
||||
1. Configure it in `puter/config/config.json`:
|
||||
|
||||
```json
|
||||
"userEmail": {
|
||||
"secret": "a long random string",
|
||||
"localServer": true,
|
||||
"localDomains": ["example.com"]
|
||||
}
|
||||
```
|
||||
|
||||
`secret` is shared between the two halves; generate it with
|
||||
`openssl rand -hex 32`. `localDomains` lists the domains this server accepts
|
||||
mail for — anything else is refused at `RCPT TO`, which is what stops it
|
||||
being used as an open relay. It has no default, and the receiver refuses to
|
||||
start without it.
|
||||
|
||||
The receiver posts to `<api_base_url>/email/ingress` unless you set
|
||||
`localIngressUrl`. The endpoint is served on the `api` subdomain, so if you
|
||||
point it at an internal address to skip the reverse proxy, name that virtual
|
||||
host too or the request will 404:
|
||||
|
||||
```json
|
||||
"localIngressUrl": "http://puter:4100/email/ingress",
|
||||
"localIngressHost": "api.example.com"
|
||||
```
|
||||
|
||||
`localIngressHost` defaults to the host of `api_base_url`, which is usually
|
||||
what you want.
|
||||
|
||||
2. Bring it up with the `smtp` profile:
|
||||
|
||||
```bash
|
||||
docker compose --profile smtp up -d
|
||||
docker compose logs -f puter-smtp
|
||||
```
|
||||
|
||||
3. Point DNS at it:
|
||||
|
||||
```dns
|
||||
mx.example.com. A 203.0.113.10
|
||||
example.com. MX 10 mx.example.com.
|
||||
```
|
||||
|
||||
Also set a `PTR` record for the address that resolves back to
|
||||
`mx.example.com` — many senders refuse mail from a server without one.
|
||||
|
||||
Inbound TCP port 25 must reach the host. **Most cloud providers block port
|
||||
25 by default and residential ISPs block it permanently**; if you cannot get
|
||||
it unblocked, this will not work no matter how it is configured. The
|
||||
container itself binds 2525 (it runs unprivileged) and compose publishes
|
||||
`${SMTP_PORT:-25}` onto it, so you can move the host port if you need to.
|
||||
|
||||
A message is held in memory while it is being delivered, so peak memory is
|
||||
bounded by `localMaxClients` (default 20) times the 25 MiB message ceiling.
|
||||
Lower `localMaxClients` on a small host. There is no queue and no retry
|
||||
schedule — if delivery fails, the sending server is told to try again later and
|
||||
retrying is its job.
|
||||
|
||||
Mail is stored as `message/rfc822` objects under `~/.mail/objects/<date>/` and
|
||||
read back through `puter.email` in the SDK.
|
||||
|
||||
Every setting is documented in
|
||||
[config.template.jsonc](../config.template.jsonc). Two deliberate limits, both
|
||||
in the name of keeping the self-hosted path simple: it does not offer STARTTLS,
|
||||
so mail arrives in the clear, and it applies no per-sender rate limiting. Put a
|
||||
filtering relay in front if either matters to you.
|
||||
|
||||
Note this is a *receiving* server only: it never sends, and it must not be used
|
||||
for authenticated submission. Outbound transactional mail is the separate
|
||||
`email` block above.
|
||||
|
||||
## Building from source instead of pulling
|
||||
|
||||
To run a local build against the full stack, use a source checkout and complete
|
||||
the configuration steps above. Create `docker-compose.override.yml` in the
|
||||
repository root, next to [docker-compose.yml](../docker-compose.yml):
|
||||
|
||||
```yaml
|
||||
services:
|
||||
puter:
|
||||
pull_policy: never
|
||||
build:
|
||||
context: .
|
||||
```
|
||||
|
||||
If that file already exists, merge these settings into its `puter` service.
|
||||
Compose merges the override automatically: `build.context: .` builds this
|
||||
checkout, and `pull_policy: never` stops it pulling the published image. Git
|
||||
ignores the override, so local build settings stay out of pull requests and
|
||||
don't conflict with `docker-compose.yml` when you pull updates.
|
||||
|
||||
From the repository root, build and start the stack:
|
||||
If you want to test local Dockerfile changes against the full stack, uncomment the `build:` block in [docker-compose.yml](../docker-compose.yml) under the `puter` service, change `pull_policy: always` → `pull_policy: never`, then:
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Run the same command after changing or pulling source code to rebuild the image.
|
||||
To return to the published image, remove the local build settings (or the override
|
||||
file if those are its only settings), then run `docker compose up -d`.
|
||||
|
||||
---
|
||||
|
||||
## Managing running backend
|
||||
@@ -681,22 +410,14 @@ Migrations re-apply idempotently across pulls. Volumes are preserved.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**`missing required command: docker` — the installer stops immediately.**
|
||||
Docker isn't on your `PATH` yet. Puter self-hosting is a Docker Compose stack, so
|
||||
nothing can run without it. Install [Docker Desktop](https://docs.docker.com/get-docker/)
|
||||
on macOS/Windows or [Docker Engine](https://docs.docker.com/engine/install/) plus
|
||||
`docker-compose-plugin` on Linux, then confirm `docker compose version` works in
|
||||
the same shell and re-run the installer. On Linux, a fresh Docker install usually
|
||||
needs a new login session before your user picks up the `docker` group.
|
||||
|
||||
**Site loads but I get a 502 / "Bad Gateway" from Caddy.**
|
||||
**Site loads but I get "Bad Gateway" / nginx errors.**
|
||||
The puter container failed to come up. `docker compose logs puter` will tell you which dependency rejected it (most often DB password mismatch between `.env` and `config.json`).
|
||||
|
||||
**Can't find the admin password.**
|
||||
It's logged on every boot until you change it: `docker compose logs puter | grep "password for admin"`. After login, change it in Settings.
|
||||
**Login screen says "admin password not set".**
|
||||
First-boot temp password is logged once. Find it: `docker compose logs puter | grep "tmp_password"`. After login, change it in Settings.
|
||||
|
||||
**Healthcheck reports unhealthy but the site works.**
|
||||
The compose healthcheck requests `http://localhost:4100/` from inside the container. If you changed `port` or set `allow_all_host_values: false`, the check fails while the site itself is fine.
|
||||
The healthcheck hits `puter.localhost:4100/test` from inside the container. If you changed `domain` or `port`, the check still uses defaults. The site itself is fine.
|
||||
|
||||
**Nothing resolves at `puter.example.com` after DNS changes.**
|
||||
DNS propagates slowly. `dig puter.example.com` and `dig api.puter.example.com` should both return your server IP. If not, give it 5–60 minutes.
|
||||
|
||||
+24
-75
@@ -2,7 +2,7 @@
|
||||
# Self-hosted Puter — full stack.
|
||||
#
|
||||
# Brings up Puter + every external service it needs:
|
||||
# - caddy : reverse proxy (mirrors prod ALB; handles TLS + Host fan-out)
|
||||
# - nginx : reverse proxy (mirrors prod ALB; handles TLS + Host fan-out)
|
||||
# - valkey : redis-compatible cache / rate-limiter backend
|
||||
# - mariadb : SQL database (Puter applies its schema on first boot)
|
||||
# - dynamo : DynamoDB-local (KV store; Puter creates the table itself)
|
||||
@@ -23,8 +23,7 @@
|
||||
#
|
||||
# Production:
|
||||
# - Always replace the default passwords / S3 keys / Puter secrets.
|
||||
# - Enable TLS on the bundled Caddy (see caddy/Caddyfile) or front the
|
||||
# stack with your own TLS-terminating proxy.
|
||||
# - Front Puter with TLS-terminating reverse proxy (Caddy / nginx).
|
||||
# - Move state-bearing volumes to a backed-up location.
|
||||
|
||||
services:
|
||||
@@ -119,7 +118,7 @@ services:
|
||||
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:-puter-secret-change-me}
|
||||
volumes:
|
||||
- ./puter/data/s3:/data:z
|
||||
# Internal-only — browsers reach RustFS via Caddy (`s3.<domain>`),
|
||||
# Internal-only — browsers reach RustFS via nginx (`s3.<domain>`),
|
||||
# which preserves the Host header for S3 signature validation and
|
||||
# rides the same TLS termination as Puter. Uncomment to also expose
|
||||
# 9000 directly on the host for `aws-cli` / debugging.
|
||||
@@ -139,10 +138,8 @@ services:
|
||||
start_period: 5s
|
||||
|
||||
s3-init:
|
||||
# One-shot container that creates the `puter-local` bucket and
|
||||
# applies its CORS rules on every boot (put-bucket-cors is
|
||||
# idempotent). Without CORS, browser uploads to presigned URLs
|
||||
# fail the preflight check.
|
||||
# One-shot container that creates the `puter-local` bucket on first
|
||||
# boot. Exits 0 once the bucket exists; stays exited 0 thereafter.
|
||||
image: amazon/aws-cli:latest
|
||||
container_name: puter-s3-init
|
||||
depends_on:
|
||||
@@ -165,26 +162,6 @@ services:
|
||||
echo "creating bucket $$bucket"
|
||||
aws --endpoint-url "$$endpoint" s3 mb "s3://$$bucket"
|
||||
fi
|
||||
# Browser uploads (Dev Center deploy, puter.fs.upload) PUT to
|
||||
# presigned URLs cross-origin, so the bucket must answer the
|
||||
# OPTIONS preflight. Origins are open because the presigned URL
|
||||
# itself is the credential and the SDK sends no cookies.
|
||||
cat > /tmp/cors.json <<'EOF'
|
||||
{
|
||||
"CORSRules": [
|
||||
{
|
||||
"AllowedOrigins": ["*"],
|
||||
"AllowedMethods": ["GET", "HEAD", "PUT", "POST", "DELETE"],
|
||||
"AllowedHeaders": ["*"],
|
||||
"ExposeHeaders": ["ETag", "x-amz-request-id"],
|
||||
"MaxAgeSeconds": 3600
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
aws --endpoint-url "$$endpoint" s3api put-bucket-cors \
|
||||
--bucket "$$bucket" --cors-configuration file:///tmp/cors.json
|
||||
echo "bucket CORS configured"
|
||||
restart: "no"
|
||||
|
||||
# ── Optional: local LLM ───────────────────────────────────────────
|
||||
@@ -252,8 +229,15 @@ services:
|
||||
puter:
|
||||
image: ghcr.io/heyputer/puter:main
|
||||
pull_policy: always
|
||||
# For local builds, create a docker-compose.override.yml;
|
||||
# see doc/self-hosting.md#building-from-source-instead-of-pulling.
|
||||
# Uncomment to build from this directory instead of pulling the published
|
||||
# image. Also flip pull_policy to `never` so compose doesn't overwrite
|
||||
# your local build by re-pulling :latest.
|
||||
# build:
|
||||
# context: .
|
||||
# # buildx-only: cross-compile to both archs in a single push
|
||||
# # platforms:
|
||||
# # - linux/amd64
|
||||
# # - linux/arm64
|
||||
container_name: puter
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
@@ -265,7 +249,7 @@ services:
|
||||
condition: service_started
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
# Internal-only: Caddy reaches it on the compose network. Uncomment
|
||||
# Internal-only: nginx reaches it on the compose network. Uncomment
|
||||
# to also expose port 4100 directly on the host (useful for debugging).
|
||||
# ports:
|
||||
# - "4100:4100"
|
||||
@@ -280,64 +264,29 @@ services:
|
||||
# Persistent runtime data (anything your config points at /var/puter).
|
||||
- ./puter/data/puter:/var/puter:z
|
||||
healthcheck:
|
||||
test: wget --no-verbose --tries=1 --spider http://localhost:4100/ || exit 1
|
||||
test: wget --no-verbose --tries=1 --spider http://puter.localhost:4100/test || exit 1
|
||||
interval: 30s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
# Inbound mail. Behind the `smtp` profile, so it only starts when asked for:
|
||||
# docker compose --profile smtp up -d
|
||||
# Accepts SMTP from other mail servers and hands each message to the ingress
|
||||
# endpoint, which files it into the recipient's mailbox. Needs `userEmail`
|
||||
# configured — see doc/self-hosting.md#optional-receiving-mail-over-smtp.
|
||||
puter-smtp:
|
||||
profiles: ["smtp"]
|
||||
image: ghcr.io/heyputer/puter:main
|
||||
pull_policy: always
|
||||
container_name: puter-smtp
|
||||
restart: unless-stopped
|
||||
command: ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/smtp/index.js"]
|
||||
depends_on:
|
||||
puter:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
# The container runs unprivileged and binds 2525; 25 is what an MX record
|
||||
# means, and many providers block it.
|
||||
- "${SMTP_PORT:-25}:2525"
|
||||
environment:
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
volumes:
|
||||
# The same config.json the backend reads; only `userEmail` applies here.
|
||||
- ./puter/config:/etc/puter:z
|
||||
|
||||
caddy:
|
||||
image: caddy:2.11-alpine
|
||||
container_name: puter-caddy
|
||||
nginx:
|
||||
image: nginx:1.27-alpine
|
||||
container_name: puter-nginx
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
puter:
|
||||
condition: service_started
|
||||
ports:
|
||||
- "${HTTP_PORT:-80}:80"
|
||||
# Uncomment when you enable TLS in caddy/Caddyfile:
|
||||
# Uncomment when you enable TLS in nginx/nginx.conf:
|
||||
# - "${HTTPS_PORT:-443}:443"
|
||||
volumes:
|
||||
- ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro,z
|
||||
# TLS certs (wildcard fullchain.pem + privkey.pem). Read-only inside.
|
||||
- ./puter/tls:/etc/caddy/tls:ro,z
|
||||
# Caddy's own state — the cert store it would use if you swap this
|
||||
# config over to ACME. Kept under ./puter/data with everything else
|
||||
# so one directory is the whole backup.
|
||||
- ./puter/data/caddy:/data:z
|
||||
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro,z
|
||||
# TLS certs (fullchain.pem + privkey.pem). Read-only inside.
|
||||
- ./puter/tls:/etc/nginx/tls:ro,z
|
||||
healthcheck:
|
||||
# Hits Caddy's local-only admin API rather than proxying through to
|
||||
# Puter: this reports whether the proxy itself is up and configured,
|
||||
# and keeps working once the `:80` block becomes an HTTPS redirect.
|
||||
# 127.0.0.1, not localhost — the admin endpoint is IPv4-only and
|
||||
# busybox wget tries ::1 first.
|
||||
test: ["CMD-SHELL", "wget -qO- --tries=1 --timeout=2 http://127.0.0.1:2019/config/ >/dev/null || exit 1"]
|
||||
test: ["CMD-SHELL", "wget -qO- --tries=1 --timeout=2 http://localhost/ || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
+142
-146
@@ -1,92 +1,75 @@
|
||||
import type { Request, Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
afterAll,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { runWithContext } from '../src/backend/core/context.ts';
|
||||
import { PuterServer } from '../src/backend/server.ts';
|
||||
import { setupTestServer } from '../src/backend/testUtil.ts';
|
||||
// Importing the module registers the `appTelemetry` driver into the shared
|
||||
// extensionStore, so `setupTestServer` instantiates it on `server.drivers`.
|
||||
import { AppTelemetryDriver } from './appTelemetry.ts';
|
||||
import {
|
||||
handleAppTelemetryUserCount,
|
||||
handleAppTelemetryUsers,
|
||||
} from './appTelemetry.ts';
|
||||
|
||||
interface CapturedResponse {
|
||||
body: unknown;
|
||||
}
|
||||
|
||||
const makeReq = (query: Record<string, unknown> = {}): Request =>
|
||||
({ query }) as unknown as Request;
|
||||
|
||||
const makeRes = () => {
|
||||
const captured: CapturedResponse = { body: undefined };
|
||||
const res = {
|
||||
json: vi.fn((value: unknown) => {
|
||||
captured.body = value;
|
||||
return res;
|
||||
}),
|
||||
status: vi.fn(() => res),
|
||||
};
|
||||
return { res: res as unknown as Response, captured };
|
||||
};
|
||||
|
||||
// `handleAppTelemetryUsers` calls `Context.get('actor')` for the permission
|
||||
// gate. Wrap test calls in `runWithContext` so the ALS lookup resolves.
|
||||
const callWithActor = async (
|
||||
actor: { user: { uuid: string; id?: number }; app?: null } | undefined,
|
||||
fn: () => Promise<void>,
|
||||
) => runWithContext({ actor }, fn);
|
||||
|
||||
let server: PuterServer;
|
||||
let driver: AppTelemetryDriver;
|
||||
|
||||
// `get_users` reads `Context.get('actor')` for the ownership gate. Wrap
|
||||
// actor-dependent calls in `runWithContext` so the ALS lookup resolves.
|
||||
const callWithActor = <T>(
|
||||
actor: { user: { uuid: string; id?: number } } | undefined,
|
||||
fn: () => Promise<T>,
|
||||
) => runWithContext({ actor }, fn);
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
driver = (server.drivers as unknown as Record<string, AppTelemetryDriver>)
|
||||
.appTelemetry;
|
||||
// Guard: the driver must have been wired onto the server. If this is
|
||||
// undefined the extension didn't register, and every test below would
|
||||
// fail with a confusing "cannot read property of undefined".
|
||||
expect(driver).toBeInstanceOf(AppTelemetryDriver);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
await server.shutdown();
|
||||
});
|
||||
|
||||
const seedOwnedApp = async (prefix: string) => {
|
||||
const owner = await server.stores.user.create({
|
||||
username: `${prefix}_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: 'x',
|
||||
email: null,
|
||||
});
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `${prefix}_${slug}`,
|
||||
title: `${prefix} ${slug}`,
|
||||
index_url: `https://example.com/${slug}`,
|
||||
},
|
||||
{ ownerUserId: owner.id as number },
|
||||
);
|
||||
return { owner, app: app! };
|
||||
};
|
||||
|
||||
const seedUser = async (prefix: string, email: string | null = null) =>
|
||||
server.stores.user.create({
|
||||
username: `${prefix}_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: 'x',
|
||||
email,
|
||||
});
|
||||
|
||||
const grantAuthenticated = async (appId: number, userId: number) => {
|
||||
await server.clients.db.write(
|
||||
`INSERT INTO user_to_app_permissions (user_id, app_id, permission, extra) VALUES (?, ?, ?, ?)`,
|
||||
[userId, appId, 'flag:app-is-authenticated', null],
|
||||
);
|
||||
};
|
||||
|
||||
const grantEmailRead = async (
|
||||
appId: number,
|
||||
userId: number,
|
||||
userUuid: string,
|
||||
) => {
|
||||
await server.clients.db.write(
|
||||
`INSERT INTO user_to_app_permissions (user_id, app_id, permission, extra) VALUES (?, ?, ?, ?)`,
|
||||
[userId, appId, `user:${userUuid}:email:read`, null],
|
||||
);
|
||||
};
|
||||
|
||||
describe('appTelemetry driver — get_users', () => {
|
||||
describe('appTelemetry extension — handleAppTelemetryUsers', () => {
|
||||
it('throws HttpError(400) when app_uuid is missing', async () => {
|
||||
await expect(driver.get_users({})).rejects.toMatchObject({
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
handleAppTelemetryUsers(makeReq({}), res),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
message: expect.stringContaining('app_uuid'),
|
||||
});
|
||||
});
|
||||
|
||||
it('throws HttpError(400) for a non-numeric limit', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
driver.get_users({ app_uuid: 'app-anything', limit: 'banana' }),
|
||||
handleAppTelemetryUsers(
|
||||
makeReq({ app_uuid: 'app-anything', limit: 'banana' }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
message: expect.stringContaining('limit'),
|
||||
@@ -94,14 +77,22 @@ describe('appTelemetry driver — get_users', () => {
|
||||
});
|
||||
|
||||
it('throws HttpError(400) when offset exceeds the allowed maximum', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
driver.get_users({ app_uuid: 'app-anything', offset: 1_000_001 }),
|
||||
handleAppTelemetryUsers(
|
||||
makeReq({ app_uuid: 'app-anything', offset: 1_000_001 }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('throws HttpError(404) when the app cannot be found', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
driver.get_users({ app_uuid: 'app-does-not-exist' }),
|
||||
handleAppTelemetryUsers(
|
||||
makeReq({ app_uuid: 'app-does-not-exist' }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 404,
|
||||
message: 'App not found',
|
||||
@@ -109,10 +100,25 @@ describe('appTelemetry driver — get_users', () => {
|
||||
});
|
||||
|
||||
it('throws HttpError(403) when the caller does not own the app', async () => {
|
||||
// Seed an owner + their app, then call as a *different* user. The
|
||||
// real permission service should reject: the caller has no
|
||||
// `apps-of-user:<owner>:write`.
|
||||
const { app } = await seedOwnedApp('owner');
|
||||
// Seed an owner user + an app belonging to them, then call the
|
||||
// handler as a *different* user. The real permission service should
|
||||
// reject since the caller has no `apps-of-user:<owner>:write` perm.
|
||||
const owner = await server.stores.user.create({
|
||||
username: `owner_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: 'x',
|
||||
email: null,
|
||||
});
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `app_${slug}`,
|
||||
title: `App ${slug}`,
|
||||
index_url: `https://example.com/${slug}`,
|
||||
},
|
||||
{ ownerUserId: owner.id as number },
|
||||
);
|
||||
|
||||
const intruder = await server.stores.user.create({
|
||||
username: `intruder_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
@@ -120,11 +126,15 @@ describe('appTelemetry driver — get_users', () => {
|
||||
email: null,
|
||||
});
|
||||
|
||||
const { res } = makeRes();
|
||||
await callWithActor(
|
||||
{ user: { uuid: intruder.uuid, id: intruder.id as number } },
|
||||
async () => {
|
||||
await expect(
|
||||
driver.get_users({ app_uuid: app.uid }),
|
||||
handleAppTelemetryUsers(
|
||||
makeReq({ app_uuid: app!.uid }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
message: 'Permission denied',
|
||||
@@ -134,92 +144,78 @@ describe('appTelemetry driver — get_users', () => {
|
||||
});
|
||||
|
||||
it('returns an empty list for an owned app with no authenticated users', async () => {
|
||||
const { owner, app } = await seedOwnedApp('owner2');
|
||||
const owner = await server.stores.user.create({
|
||||
username: `owner2_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: 'x',
|
||||
email: null,
|
||||
});
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `app2_${slug}`,
|
||||
title: `App2 ${slug}`,
|
||||
index_url: `https://example.com/${slug}`,
|
||||
},
|
||||
{ ownerUserId: owner.id as number },
|
||||
);
|
||||
|
||||
const result = await callWithActor(
|
||||
const { res, captured } = makeRes();
|
||||
await callWithActor(
|
||||
{ user: { uuid: owner.uuid, id: owner.id as number } },
|
||||
() => driver.get_users({ app_uuid: app.uid }),
|
||||
async () => {
|
||||
await handleAppTelemetryUsers(
|
||||
makeReq({ app_uuid: app!.uid }),
|
||||
res,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
|
||||
it('omits user_email for a user who did not grant email:read', async () => {
|
||||
const { owner, app } = await seedOwnedApp('noemail');
|
||||
const member = await seedUser('member', '[email protected]');
|
||||
await grantAuthenticated(app.id as number, member.id as number);
|
||||
|
||||
const [row] = (await callWithActor(
|
||||
{ user: { uuid: owner.uuid, id: owner.id as number } },
|
||||
() => driver.get_users({ app_uuid: app.uid }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
|
||||
expect(row.user).toBe(member.username);
|
||||
expect(row.user_uuid).toBe(member.uuid);
|
||||
// No grant → the field must be absent (not just null), so the email
|
||||
// never leaks to the app owner.
|
||||
expect(Object.prototype.hasOwnProperty.call(row, 'user_email')).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('returns user_email when the user granted email:read to the app', async () => {
|
||||
const { owner, app } = await seedOwnedApp('withemail');
|
||||
const member = await seedUser('member', '[email protected]');
|
||||
await grantAuthenticated(app.id as number, member.id as number);
|
||||
await grantEmailRead(
|
||||
app.id as number,
|
||||
member.id as number,
|
||||
member.uuid,
|
||||
);
|
||||
|
||||
const [row] = (await callWithActor(
|
||||
{ user: { uuid: owner.uuid, id: owner.id as number } },
|
||||
() => driver.get_users({ app_uuid: app.uid }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
|
||||
expect(row.user_uuid).toBe(member.uuid);
|
||||
expect(row.user_email).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('does not leak email granted to a *different* app', async () => {
|
||||
const { owner, app } = await seedOwnedApp('appA');
|
||||
const { app: otherApp } = await seedOwnedApp('appB');
|
||||
const member = await seedUser('member', '[email protected]');
|
||||
await grantAuthenticated(app.id as number, member.id as number);
|
||||
// Grant email:read against the OTHER app only.
|
||||
await grantEmailRead(
|
||||
otherApp.id as number,
|
||||
member.id as number,
|
||||
member.uuid,
|
||||
);
|
||||
|
||||
const [row] = (await callWithActor(
|
||||
{ user: { uuid: owner.uuid, id: owner.id as number } },
|
||||
() => driver.get_users({ app_uuid: app.uid }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
|
||||
expect(Object.prototype.hasOwnProperty.call(row, 'user_email')).toBe(
|
||||
false,
|
||||
);
|
||||
expect(captured.body).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('appTelemetry driver — user_count', () => {
|
||||
describe('appTelemetry extension — handleAppTelemetryUserCount', () => {
|
||||
it('throws HttpError(400) when app_uuid is missing', async () => {
|
||||
await expect(driver.user_count({})).rejects.toMatchObject({
|
||||
statusCode: 400,
|
||||
});
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
handleAppTelemetryUserCount(makeReq({}), res),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('throws HttpError(404) for an unknown app_uuid', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
driver.user_count({ app_uuid: 'app-not-here' }),
|
||||
handleAppTelemetryUserCount(
|
||||
makeReq({ app_uuid: 'app-not-here' }),
|
||||
res,
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('returns 0 for an app with no authenticated users', async () => {
|
||||
const { app } = await seedOwnedApp('appcount');
|
||||
await expect(driver.user_count({ app_uuid: app.uid })).resolves.toBe(0);
|
||||
it('returns { count: 0 } for an app with no authenticated users', async () => {
|
||||
const owner = await server.stores.user.create({
|
||||
username: `counto_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: 'x',
|
||||
email: null,
|
||||
});
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
const app = await server.stores.app.create(
|
||||
{
|
||||
name: `appcount_${slug}`,
|
||||
title: `AppCount ${slug}`,
|
||||
index_url: `https://example.com/${slug}`,
|
||||
},
|
||||
{ ownerUserId: owner.id as number },
|
||||
);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await handleAppTelemetryUserCount(
|
||||
makeReq({ app_uuid: app!.uid }),
|
||||
res,
|
||||
);
|
||||
|
||||
expect(captured.body).toEqual({ count: 0 });
|
||||
});
|
||||
});
|
||||
+80
-124
@@ -1,23 +1,11 @@
|
||||
import type { Request, Response } from 'express';
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { HttpError } from '@heyputer/backend/src/core/http';
|
||||
import { PuterDriver } from '@heyputer/backend/src/drivers/types';
|
||||
import type {
|
||||
DriverConcurrentConfig,
|
||||
DriverRateLimitConfig,
|
||||
} from '@heyputer/backend/src/drivers/meta';
|
||||
import {
|
||||
DEFAULT_FREE_SUBSCRIPTION,
|
||||
DEFAULT_TEMP_SUBSCRIPTION,
|
||||
} from '@heyputer/backend/src/services/metering/consts';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
|
||||
// App-telemetry lets an app owner enumerate the users who have
|
||||
// authenticated into their app. v1 shipped this as a driver on the
|
||||
// `app-telemetry` interface (methods `get_users` / `user_count`) and
|
||||
// puter-js's `puter.apps(...).getUsers()` still calls it that way
|
||||
// (`puter.drivers.call('app-telemetry', 'app-telemetry', 'get_users', …)`).
|
||||
// This is the v2 port of that driver — same interface/method/return shapes
|
||||
// so existing puter-js callers work unchanged.
|
||||
const clients = extension.import('client');
|
||||
const stores = extension.import('store');
|
||||
const services = extension.import('service');
|
||||
|
||||
const DEFAULT_LIMIT = 100;
|
||||
const MAX_LIMIT = 1000;
|
||||
@@ -53,123 +41,91 @@ const parseIntParam = (
|
||||
return parsed;
|
||||
};
|
||||
|
||||
/**
|
||||
* Driver exposing the `app-telemetry` interface.
|
||||
*
|
||||
* The `/drivers/call` permission gate checks
|
||||
* `service:app-telemetry:ii:app-telemetry`, which every actor already holds via
|
||||
* the blanket `service` grant (hardcoded-permissions.js +
|
||||
* `default_implicit_user_app_permissions`). The real authorization — "is the
|
||||
* caller the app owner?" — is enforced inside `get_users` below, exactly as v1
|
||||
* did.
|
||||
*/
|
||||
export class AppTelemetryDriver extends PuterDriver {
|
||||
readonly driverInterface = 'app-telemetry';
|
||||
readonly driverName = 'app-telemetry';
|
||||
readonly isDefault = true;
|
||||
export const handleAppTelemetryUsers = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
const { app_uuid } = req.query as Record<string, string>;
|
||||
if (!app_uuid) throw new HttpError(400, 'Missing `app_uuid`');
|
||||
|
||||
// Declaring nothing here would leave both methods on the generic
|
||||
// 600/minute driver default, which does not fit a paginated scan that
|
||||
// can ask for MAX_LIMIT rows at MAX_OFFSET. This is a dashboard read —
|
||||
// nobody calls it in a loop.
|
||||
readonly rateLimit: DriverRateLimitConfig = {
|
||||
default: {
|
||||
limit: 60,
|
||||
window: 60_000,
|
||||
bySubscription: {
|
||||
[DEFAULT_FREE_SUBSCRIPTION]: 30,
|
||||
[DEFAULT_TEMP_SUBSCRIPTION]: 10,
|
||||
},
|
||||
},
|
||||
};
|
||||
const safeLimit = parseIntParam(req.query.limit, {
|
||||
key: 'limit',
|
||||
min: 1,
|
||||
max: MAX_LIMIT,
|
||||
fallback: DEFAULT_LIMIT,
|
||||
});
|
||||
const safeOffset = parseIntParam(req.query.offset, {
|
||||
key: 'offset',
|
||||
min: 0,
|
||||
max: MAX_OFFSET,
|
||||
fallback: 0,
|
||||
});
|
||||
|
||||
readonly concurrent: DriverConcurrentConfig = {
|
||||
default: {
|
||||
limit: 5,
|
||||
bySubscription: {
|
||||
[DEFAULT_FREE_SUBSCRIPTION]: 2,
|
||||
[DEFAULT_TEMP_SUBSCRIPTION]: 2,
|
||||
},
|
||||
},
|
||||
};
|
||||
const app = await stores.app.getByUid(app_uuid);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
|
||||
/** Users who have authenticated into the given app (owner-only). */
|
||||
async get_users({
|
||||
app_uuid,
|
||||
limit,
|
||||
offset,
|
||||
}: {
|
||||
app_uuid?: string;
|
||||
limit?: unknown;
|
||||
offset?: unknown;
|
||||
} = {}): Promise<
|
||||
Array<{ user: string; user_uuid: string; user_email?: string | null }>
|
||||
> {
|
||||
if (!app_uuid) throw new HttpError(400, 'Missing `app_uuid`');
|
||||
// `apps-of-user:<uuid>:write` — the implicator keys on the owner's
|
||||
// UUID, not the numeric id. Look up the owner explicitly. v1 got
|
||||
// this for free because its entity-storage layer eager-joined the
|
||||
// owner row; v2's AppStore.getByUid returns the raw row with only
|
||||
// `owner_user_id` populated.
|
||||
const ownerId = (app as { owner_user_id?: number }).owner_user_id;
|
||||
if (!ownerId) throw new HttpError(404, 'App owner not found');
|
||||
const owner = (await stores.user.getById(ownerId)) as {
|
||||
uuid?: string;
|
||||
} | null;
|
||||
if (!owner?.uuid) throw new HttpError(404, 'App owner not found');
|
||||
|
||||
const safeLimit = parseIntParam(limit, {
|
||||
key: 'limit',
|
||||
min: 1,
|
||||
max: MAX_LIMIT,
|
||||
fallback: DEFAULT_LIMIT,
|
||||
});
|
||||
const safeOffset = parseIntParam(offset, {
|
||||
key: 'offset',
|
||||
min: 0,
|
||||
max: MAX_OFFSET,
|
||||
fallback: 0,
|
||||
});
|
||||
const actor = Context.get('actor');
|
||||
const ownsApp = await services.permission
|
||||
.check(actor!, `apps-of-user:${owner.uuid}:write`)
|
||||
.catch(() => false);
|
||||
if (!ownsApp) throw new HttpError(403, 'Permission denied');
|
||||
|
||||
const app = await this.stores.app.getByUid(app_uuid);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
const users = await clients.db.read(
|
||||
`SELECT u.username, u.uuid FROM user_to_app_permissions p
|
||||
INNER JOIN user u ON p.user_id = u.id
|
||||
WHERE p.permission = 'flag:app-is-authenticated' AND p.app_id = ?
|
||||
ORDER BY (p.dt IS NOT NULL), p.dt, p.user_id
|
||||
LIMIT ? OFFSET ?`,
|
||||
[(app as Record<string, unknown>).id, safeLimit, safeOffset],
|
||||
);
|
||||
|
||||
// The `apps-of-user:<uuid>:write` implicator keys on the owner's
|
||||
// UUID, not the numeric id. Look up the owner explicitly — the raw
|
||||
// app row only carries `owner_user_id`. (v1 got the owner for free
|
||||
// because its entity-storage layer eager-joined the owner row.)
|
||||
const ownerId = (app as { owner_user_id?: number }).owner_user_id;
|
||||
if (!ownerId) throw new HttpError(404, 'App owner not found');
|
||||
const owner = await this.stores.user.getById(ownerId);
|
||||
if (!owner?.uuid) throw new HttpError(404, 'App owner not found');
|
||||
res.json(
|
||||
(users as Array<{ username: string; uuid: string }>).map((e) => ({
|
||||
user: e.username,
|
||||
user_uuid: e.uuid,
|
||||
})),
|
||||
);
|
||||
};
|
||||
|
||||
// `/drivers/call` requires auth, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
let ownsApp = false;
|
||||
try {
|
||||
ownsApp = await this.services.permission.check(
|
||||
actor,
|
||||
`apps-of-user:${owner.uuid}:write`,
|
||||
);
|
||||
} catch {
|
||||
// A failed check is a denial.
|
||||
}
|
||||
if (!ownsApp) throw new HttpError(403, 'Permission denied');
|
||||
export const handleAppTelemetryUserCount = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
const { app_uuid } = req.query as Record<string, string>;
|
||||
if (!app_uuid) throw new HttpError(400, 'Missing `app_uuid`');
|
||||
|
||||
// An email surfaces only for a user who granted *this app*
|
||||
// `user:<their-uuid>:email:read` — the grant `puter.perms.requestEmail()`
|
||||
// obtains and `whoami` honours. Authenticating alone doesn't share it.
|
||||
const users = await this.stores.permission.listAppAuthenticatedUsers(
|
||||
(app as { id: number }).id,
|
||||
{ limit: safeLimit, offset: safeOffset },
|
||||
);
|
||||
return users.map((u) =>
|
||||
u.emailShared
|
||||
? { user: u.username, user_uuid: u.uuid, user_email: u.email }
|
||||
: { user: u.username, user_uuid: u.uuid },
|
||||
);
|
||||
}
|
||||
const app = await stores.app.getByUid(app_uuid);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
|
||||
/** Count of users who have authenticated into the given app. */
|
||||
async user_count({
|
||||
app_uuid,
|
||||
}: { app_uuid?: string } = {}): Promise<number> {
|
||||
if (!app_uuid) throw new HttpError(400, 'Missing `app_uuid`');
|
||||
const [row] = (await clients.db.read(
|
||||
`SELECT COUNT(*) AS n FROM user_to_app_permissions
|
||||
WHERE permission = 'flag:app-is-authenticated' AND app_id = ?`,
|
||||
[(app as Record<string, unknown>).id],
|
||||
)) as Array<{ n: number }>;
|
||||
|
||||
const app = await this.stores.app.getByUid(app_uuid);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
res.json({ count: row?.n ?? 0 });
|
||||
};
|
||||
|
||||
return this.stores.permission.countAppUsers((app as { id: number }).id);
|
||||
}
|
||||
}
|
||||
extension.get(
|
||||
'/app-telemetry/users',
|
||||
{ subdomain: 'api', requireAuth: true },
|
||||
handleAppTelemetryUsers,
|
||||
);
|
||||
|
||||
extension.registerDriver('appTelemetry', AppTelemetryDriver);
|
||||
extension.get(
|
||||
'/app-telemetry/user-count',
|
||||
{ subdomain: 'api', requireAuth: true },
|
||||
handleAppTelemetryUserCount,
|
||||
);
|
||||
@@ -1,555 +0,0 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { extensionStore } from '../src/backend/extensions.ts';
|
||||
import type { IConfig } from '../src/backend/types';
|
||||
import './devWatcher.ts';
|
||||
|
||||
type Lifecycle = {
|
||||
onServerStart: () => Promise<void>;
|
||||
onServerShutdown: () => Promise<void>;
|
||||
};
|
||||
|
||||
// The extension registers itself on import; that registry entry is the
|
||||
// only handle on the service class.
|
||||
const DevWatcherService = extensionStore.services.devWatcher as unknown as new (
|
||||
config: IConfig,
|
||||
clients: unknown,
|
||||
stores: unknown,
|
||||
services: unknown,
|
||||
) => Lifecycle;
|
||||
|
||||
const makeService = (config: Record<string, unknown>): Lifecycle =>
|
||||
new DevWatcherService(config as unknown as IConfig, {}, {}, {});
|
||||
|
||||
let workdir: string;
|
||||
|
||||
const write = (relative: string, contents: string): string => {
|
||||
const abs = path.join(workdir, relative);
|
||||
writeFileSync(abs, contents);
|
||||
return abs;
|
||||
};
|
||||
|
||||
/** Poll until `predicate` holds or the budget runs out. */
|
||||
const waitFor = async (
|
||||
predicate: () => boolean,
|
||||
label: string,
|
||||
timeoutMs = 15_000,
|
||||
): Promise<void> => {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (predicate()) return;
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
throw new Error(`timed out waiting for ${label}`);
|
||||
};
|
||||
|
||||
const logLines = (spy: { mock: { calls: unknown[][] } }): string[] =>
|
||||
spy.mock.calls.map((call) => call.map(String).join(' '));
|
||||
|
||||
describe('devWatcher extension', () => {
|
||||
let log: ReturnType<typeof vi.spyOn>;
|
||||
let warn: ReturnType<typeof vi.spyOn>;
|
||||
let error: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeAll(() => {
|
||||
workdir = mkdtempSync(path.join(tmpdir(), 'devwatch-test-'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(workdir, { recursive: true, force: true });
|
||||
delete (extensionStore.services as Record<string, unknown>).devWatcher;
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
error = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
log.mockRestore();
|
||||
warn.mockRestore();
|
||||
error.mockRestore();
|
||||
});
|
||||
|
||||
const started = () =>
|
||||
logLines(log).some((line) =>
|
||||
line.startsWith('[devwatch] starting watchers from'),
|
||||
);
|
||||
|
||||
// -- start gating -------------------------------------------------
|
||||
|
||||
it('does nothing when the server is not running in dev', async () => {
|
||||
await makeService({ env: 'production' }).onServerStart();
|
||||
expect(started()).toBe(false);
|
||||
});
|
||||
|
||||
it('does nothing when devwatch is explicitly disabled', async () => {
|
||||
await makeService({
|
||||
env: 'dev',
|
||||
devwatch: { enabled: false },
|
||||
}).onServerStart();
|
||||
expect(started()).toBe(false);
|
||||
});
|
||||
|
||||
it('does nothing when the server opts out with no_devwatch', async () => {
|
||||
await makeService({
|
||||
env: 'dev',
|
||||
no_devwatch: true,
|
||||
devwatch: { enabled: true },
|
||||
}).onServerStart();
|
||||
expect(started()).toBe(false);
|
||||
});
|
||||
|
||||
it('starts in dev even with no explicit devwatch block', async () => {
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
await service.onServerStart();
|
||||
expect(started()).toBe(true);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('starts outside dev when devwatch is explicitly enabled', async () => {
|
||||
const service = makeService({
|
||||
env: 'production',
|
||||
devwatch: {
|
||||
enabled: true,
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
await service.onServerStart();
|
||||
expect(started()).toBe(true);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('ignores a devwatch config that is not an object', async () => {
|
||||
// Falls back to `{}`, so the dev-env rule decides — and the
|
||||
// default webpack entries would be used, so pin it off instead.
|
||||
await makeService({
|
||||
env: 'production',
|
||||
devwatch: 'yes',
|
||||
}).onServerStart();
|
||||
expect(started()).toBe(false);
|
||||
});
|
||||
|
||||
it('only starts once even if the lifecycle hook fires again', async () => {
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
await service.onServerStart();
|
||||
await service.onServerStart();
|
||||
expect(
|
||||
logLines(log).filter((line) =>
|
||||
line.startsWith('[devwatch] starting watchers from'),
|
||||
),
|
||||
).toHaveLength(1);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('waits out the configured ready delay before resolving', async () => {
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [],
|
||||
ready_delay_ms: 60,
|
||||
},
|
||||
});
|
||||
const start = Date.now();
|
||||
await service.onServerStart();
|
||||
expect(Date.now() - start).toBeGreaterThanOrEqual(50);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
// -- child commands -----------------------------------------------
|
||||
|
||||
it('spawns a command, line-buffers its output and flushes the tail', async () => {
|
||||
write(
|
||||
'chatty.js',
|
||||
[
|
||||
"process.stdout.write('one\\ntwo\\n');",
|
||||
"process.stderr.write('bad thing\\n');",
|
||||
// No trailing newline: only the stream `end` flushes it.
|
||||
"process.stdout.write('tail-no-newline');",
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [
|
||||
{
|
||||
name: 'chatty',
|
||||
directory: '.',
|
||||
command: 'node',
|
||||
args: ['chatty.js'],
|
||||
},
|
||||
],
|
||||
webpack: [],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
await service.onServerStart();
|
||||
|
||||
await waitFor(
|
||||
() =>
|
||||
logLines(log).some((l) =>
|
||||
l.includes('[devwatch:chatty:1] tail-no-newline'),
|
||||
),
|
||||
'child stdout',
|
||||
);
|
||||
|
||||
const lines = logLines(log);
|
||||
expect(lines).toContain('[devwatch:chatty:1] one');
|
||||
expect(lines).toContain('[devwatch:chatty:1] two');
|
||||
expect(logLines(warn)).toContain('[devwatch:chatty:2] bad thing');
|
||||
|
||||
await waitFor(
|
||||
() =>
|
||||
logLines(log).some((l) =>
|
||||
l.includes('[devwatch:chatty:exit] process exited'),
|
||||
),
|
||||
'child exit',
|
||||
);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('passes literal and computed env values to the child', async () => {
|
||||
write(
|
||||
'env.js',
|
||||
'process.stdout.write(`STATIC=${process.env.STATIC_VALUE} ORIGIN=${process.env.FROM_CONFIG} MISSING=${process.env.BLOWS_UP}\\n`);',
|
||||
);
|
||||
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
origin: 'http://puter.localhost:4100',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [
|
||||
{
|
||||
name: 'envtest',
|
||||
directory: '.',
|
||||
command: 'node',
|
||||
args: ['env.js'],
|
||||
env: {
|
||||
STATIC_VALUE: 'literal',
|
||||
FROM_CONFIG: ({
|
||||
global_config,
|
||||
}: {
|
||||
global_config: Record<string, unknown> | null;
|
||||
}) => String(global_config?.origin ?? ''),
|
||||
// Reading through a null is the "config not
|
||||
// loaded yet" shape the extension deliberately
|
||||
// stays quiet about.
|
||||
BLOWS_UP: () => {
|
||||
const nothing = null as unknown as {
|
||||
x: string;
|
||||
};
|
||||
return nothing.x;
|
||||
},
|
||||
NOISY: () => {
|
||||
throw new Error('unexpected failure');
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
webpack: [],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
await service.onServerStart();
|
||||
|
||||
await waitFor(
|
||||
() => logLines(log).some((l) => l.includes('STATIC=literal')),
|
||||
'env output',
|
||||
);
|
||||
const line = logLines(log).find((l) => l.includes('STATIC=literal'))!;
|
||||
expect(line).toContain('ORIGIN=http://puter.localhost:4100');
|
||||
expect(line).toContain('MISSING=undefined');
|
||||
|
||||
// A null-property read is expected noise and stays silent; any
|
||||
// other failure is reported.
|
||||
const warnings = logLines(warn);
|
||||
expect(
|
||||
warnings.some((w) => w.includes('could not evaluate env function')),
|
||||
).toBe(true);
|
||||
expect(warnings.some((w) => w.includes('for BLOWS_UP'))).toBe(false);
|
||||
expect(warnings.some((w) => w.includes('for NOISY'))).toBe(true);
|
||||
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('kills a still-running child on shutdown', async () => {
|
||||
write('forever.js', 'setInterval(() => {}, 1000);');
|
||||
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [
|
||||
{
|
||||
name: 'forever',
|
||||
directory: '.',
|
||||
command: 'node',
|
||||
args: ['forever.js'],
|
||||
},
|
||||
],
|
||||
webpack: [],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
await service.onServerStart();
|
||||
await service.onServerShutdown();
|
||||
|
||||
expect(logLines(log)).toContain('[devwatch:forever] stopping process');
|
||||
await waitFor(
|
||||
() =>
|
||||
logLines(log).some((l) =>
|
||||
l.includes('[devwatch:forever:exit] process exited'),
|
||||
),
|
||||
'child killed',
|
||||
);
|
||||
});
|
||||
|
||||
// -- webpack watchers ---------------------------------------------
|
||||
|
||||
const makeWebpackProject = (
|
||||
name: string,
|
||||
configFile: string,
|
||||
contents: string,
|
||||
packageJson?: string,
|
||||
): string => {
|
||||
const dir = path.join(workdir, name);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
writeFileSync(
|
||||
path.join(
|
||||
(() => {
|
||||
const { mkdirSync } =
|
||||
require('node:fs') as typeof import('node:fs');
|
||||
mkdirSync(path.join(dir, 'src'), { recursive: true });
|
||||
return dir;
|
||||
})(),
|
||||
'src/entry.js',
|
||||
),
|
||||
"console.log('hello');\n",
|
||||
);
|
||||
writeFileSync(path.join(dir, configFile), contents);
|
||||
if (packageJson) {
|
||||
writeFileSync(path.join(dir, 'package.json'), packageJson);
|
||||
}
|
||||
return name;
|
||||
};
|
||||
|
||||
it('compiles a CommonJS webpack config and reports later rebuilds', async () => {
|
||||
const dir = makeWebpackProject(
|
||||
'cjs-project',
|
||||
'webpack.config.cjs',
|
||||
`module.exports = {
|
||||
mode: 'development',
|
||||
entry: './src/entry.js',
|
||||
output: { path: __dirname + '/out', filename: 'bundle.js' },
|
||||
};`,
|
||||
);
|
||||
|
||||
let onConfigSaw: Record<string, unknown> | undefined;
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [
|
||||
{
|
||||
name: 'cjs',
|
||||
directory: dir,
|
||||
onConfig: (cfg: Record<string, unknown>) => {
|
||||
onConfigSaw = cfg;
|
||||
},
|
||||
},
|
||||
],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
|
||||
await service.onServerStart();
|
||||
expect(onConfigSaw).toBeDefined();
|
||||
// The watcher context is anchored at <root>/<directory> so relative
|
||||
// entries resolve regardless of the server's cwd.
|
||||
expect(onConfigSaw!.context).toBe(path.join(workdir, dir));
|
||||
|
||||
// First build is silent by design; force a rebuild and assert the
|
||||
// update line shows up.
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
writeFileSync(
|
||||
path.join(workdir, dir, 'src/entry.js'),
|
||||
"console.log('hello again');\n",
|
||||
);
|
||||
await waitFor(
|
||||
() =>
|
||||
logLines(log).some((l) =>
|
||||
l.includes('[devwatch] updated cjs using Webpack'),
|
||||
),
|
||||
'webpack rebuild',
|
||||
);
|
||||
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('resolves an ESM webpack config declared through package.json type', async () => {
|
||||
const dir = makeWebpackProject(
|
||||
'esm-project',
|
||||
'webpack.config.js',
|
||||
`export default {
|
||||
mode: 'development',
|
||||
entry: './src/entry.js',
|
||||
output: { filename: 'bundle.js' },
|
||||
};`,
|
||||
JSON.stringify({ name: 'esm-project', type: 'module' }),
|
||||
);
|
||||
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [{ directory: dir }],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
|
||||
await service.onServerStart();
|
||||
await service.onServerShutdown();
|
||||
// Falling back to the directory as the display name is the
|
||||
// documented behaviour when `name` is omitted.
|
||||
expect(error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls a config exported as a function and honours an explicit context', async () => {
|
||||
const dir = makeWebpackProject(
|
||||
'fn-project',
|
||||
'webpack.config.cjs',
|
||||
`module.exports = () => ({
|
||||
mode: 'development',
|
||||
context: 'src',
|
||||
entry: './entry.js',
|
||||
name: process.env.WEBPACK_MARKER,
|
||||
output: { path: __dirname + '/out', filename: 'bundle.js' },
|
||||
});`,
|
||||
);
|
||||
|
||||
let seen: Record<string, unknown> | undefined;
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [
|
||||
{
|
||||
name: 'fn',
|
||||
directory: dir,
|
||||
env: { WEBPACK_MARKER: 'set-during-load' },
|
||||
onConfig: (cfg: Record<string, unknown>) => {
|
||||
seen = cfg;
|
||||
},
|
||||
},
|
||||
],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
|
||||
await service.onServerStart();
|
||||
// Relative `context` resolves against <root>/<directory>.
|
||||
expect(seen!.context).toBe(path.join(workdir, dir, 'src'));
|
||||
// The entry's env map is applied while the config module runs, and
|
||||
// restored afterwards.
|
||||
expect(seen!.name).toBe('set-during-load');
|
||||
expect(process.env.WEBPACK_MARKER).toBeUndefined();
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('reports a failing compilation instead of crashing the server', async () => {
|
||||
const dir = makeWebpackProject(
|
||||
'broken-project',
|
||||
'webpack.config.cjs',
|
||||
`module.exports = {
|
||||
mode: 'development',
|
||||
entry: './src/does-not-exist.js',
|
||||
output: { path: __dirname + '/out', filename: 'bundle.js' },
|
||||
};`,
|
||||
);
|
||||
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [{ name: 'broken', directory: dir }],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
|
||||
await service.onServerStart();
|
||||
await waitFor(
|
||||
() =>
|
||||
logLines(error).some((l) =>
|
||||
l.includes('[devwatch] failed to update broken'),
|
||||
),
|
||||
'webpack failure',
|
||||
);
|
||||
expect(
|
||||
logLines(error).some((l) =>
|
||||
l.includes('[devwatch] error information: broken'),
|
||||
),
|
||||
).toBe(true);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
|
||||
it('fails loudly when a directory has no webpack config at all', async () => {
|
||||
const dir = path.join('no-config');
|
||||
const { mkdirSync } = await import('node:fs');
|
||||
mkdirSync(path.join(workdir, dir), { recursive: true });
|
||||
|
||||
const service = makeService({
|
||||
env: 'dev',
|
||||
devwatch: {
|
||||
root: workdir,
|
||||
commands: [],
|
||||
webpack: [{ name: 'missing', directory: dir }],
|
||||
ready_delay_ms: 0,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(service.onServerStart()).rejects.toThrow(
|
||||
'could not find webpack config for: no-config',
|
||||
);
|
||||
await service.onServerShutdown();
|
||||
});
|
||||
});
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { runWithContext } from '../src/backend/core/context.ts';
|
||||
import { extension } from '../src/backend/extensions.ts';
|
||||
import { PuterServer } from '../src/backend/server.ts';
|
||||
import { setupTestServer } from '../src/backend/testUtil.ts';
|
||||
import { handleInstalledApps } from './installedApps.ts';
|
||||
@@ -39,7 +38,7 @@ beforeAll(async () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
await server.shutdown();
|
||||
});
|
||||
|
||||
const seedUser = async () => {
|
||||
@@ -52,14 +51,13 @@ const seedUser = async () => {
|
||||
});
|
||||
};
|
||||
|
||||
const seedApp = async (ownerUserId: number, overrides = {}) => {
|
||||
const seedApp = async (ownerUserId: number) => {
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
return server.stores.app.create(
|
||||
{
|
||||
name: `iaapp_${slug}`,
|
||||
title: `Installed App ${slug}`,
|
||||
index_url: `https://example.com/${slug}`,
|
||||
...overrides,
|
||||
},
|
||||
{ ownerUserId },
|
||||
);
|
||||
@@ -74,6 +72,15 @@ const grantInstalled = async (appId: number, userId: number) => {
|
||||
};
|
||||
|
||||
describe('installedApps extension — handleInstalledApps', () => {
|
||||
it('throws HttpError(401) when no actor is on the context', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
runWithContext({ actor: undefined }, () =>
|
||||
handleInstalledApps(makeReq({}), res),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it('throws HttpError(400) when orderBy is not in the allowlist', async () => {
|
||||
const user = await seedUser();
|
||||
const { res } = makeRes();
|
||||
@@ -125,78 +132,9 @@ describe('installedApps extension — handleInstalledApps', () => {
|
||||
expect(list[0].uid).toBe(app!.uid);
|
||||
expect(list[0].name).toBe(app!.name);
|
||||
expect(list[0].title).toBe(app!.title);
|
||||
// index_url is required so the dashboard can derive a hostname title
|
||||
// for anonymous (app-…) apps.
|
||||
expect(list[0].index_url).toBe(app!.index_url);
|
||||
expect(Object.prototype.hasOwnProperty.call(list[0], 'iconUrl')).toBe(
|
||||
true,
|
||||
);
|
||||
// An owned app is not external, and the raw owner id must not leak.
|
||||
expect(list[0].external).toBe(false);
|
||||
expect(
|
||||
Object.prototype.hasOwnProperty.call(list[0], 'owner_user_id'),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('pairs iconUrl with the direct subdomain URL for a generated icon', async () => {
|
||||
const user = await seedUser();
|
||||
// The resize pipeline leaves an http(s) icon column behind, which is
|
||||
// what says the sized PNG is on the icons subdomain.
|
||||
const app = await seedApp(user.id as number, {
|
||||
icon: 'https://api.puter.com/app-icon/x',
|
||||
});
|
||||
await grantInstalled(app!.id as number, user.id as number);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleInstalledApps(makeReq({}), res),
|
||||
);
|
||||
|
||||
const list = captured.body as Array<Record<string, unknown>>;
|
||||
const { protocol, static_hosting_domain } = extension.config;
|
||||
expect(list[0].iconCdnUrl).toBe(
|
||||
`${protocol}://puter-app-icons.${static_hosting_domain}/${app!.uid}-256.png`,
|
||||
);
|
||||
});
|
||||
|
||||
it('reports no subdomain URL for an app with no generated icon', async () => {
|
||||
const user = await seedUser();
|
||||
const app = await seedApp(user.id as number);
|
||||
await grantInstalled(app!.id as number, user.id as number);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleInstalledApps(makeReq({}), res),
|
||||
);
|
||||
|
||||
const list = captured.body as Array<Record<string, unknown>>;
|
||||
expect(list[0].iconCdnUrl).toBeNull();
|
||||
});
|
||||
|
||||
it('flags apps with no owner_user_id as external', async () => {
|
||||
const user = await seedUser();
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
// createFromOrigin bootstraps an app with owner_user_id = null.
|
||||
const app = await server.stores.app.createFromOrigin(
|
||||
`app-${slug}`,
|
||||
`https://external-${slug}.example.com`,
|
||||
);
|
||||
await grantInstalled(app!.id as number, user.id as number);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleInstalledApps(makeReq({}), res),
|
||||
);
|
||||
|
||||
const list = captured.body as Array<Record<string, unknown>>;
|
||||
expect(list).toHaveLength(1);
|
||||
expect(list[0].external).toBe(true);
|
||||
expect(
|
||||
Object.prototype.hasOwnProperty.call(list[0], 'owner_user_id'),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('clamps page/limit to safe ranges (page>=1, 1<=limit<=100)', async () => {
|
||||
|
||||
+47
-45
@@ -2,73 +2,75 @@ import type { Request, Response } from 'express';
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { HttpError } from '@heyputer/backend/src/core/http';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import {
|
||||
INSTALLED_APPS_ORDERS,
|
||||
type InstalledAppsOrder,
|
||||
} from '@heyputer/backend/src/stores/permission/PermissionStore.js';
|
||||
import {
|
||||
getAppIconCdnUrl,
|
||||
getAppIconUrl,
|
||||
} from '@heyputer/backend/src/util/appIcon.js';
|
||||
import { getAppIconUrl } from '@heyputer/backend/src/util/appIcon.js';
|
||||
|
||||
const stores = extension.import('store');
|
||||
const clients = extension.import('client');
|
||||
|
||||
const ALLOWED_ORDER_BY = [
|
||||
'id',
|
||||
'name',
|
||||
'uid',
|
||||
'title',
|
||||
'installed_at',
|
||||
] as const;
|
||||
const ORDER_BY_FIELD_MAP: Record<string, string> = {
|
||||
id: 'apps.id',
|
||||
name: 'apps.name',
|
||||
uid: 'apps.uid',
|
||||
title: 'apps.title',
|
||||
installed_at: 'installed_at',
|
||||
};
|
||||
|
||||
export const handleInstalledApps = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
// Behind `requireUserActor`, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
const actor = Context.get('actor');
|
||||
if (!actor?.user?.id) throw new HttpError(401, 'Authentication required');
|
||||
|
||||
const orderBy = String(
|
||||
req.query.orderBy ?? 'installed_at',
|
||||
) as InstalledAppsOrder;
|
||||
if (!INSTALLED_APPS_ORDERS.includes(orderBy)) {
|
||||
const orderBy = String(req.query.orderBy ?? 'installed_at');
|
||||
if (!(ALLOWED_ORDER_BY as readonly string[]).includes(orderBy)) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`Invalid orderBy. Allowed: ${INSTALLED_APPS_ORDERS.join(', ')}`,
|
||||
`Invalid orderBy. Allowed: ${ALLOWED_ORDER_BY.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
const page = Math.max(Number(req.query.page) || 1, 1);
|
||||
const limit = Math.min(Math.max(Number(req.query.limit) || 100, 1), 100);
|
||||
const offset = (page - 1) * limit;
|
||||
const installedApps = await stores.permission.listAppsGrantedByUser(
|
||||
actor.user.id!,
|
||||
{ orderBy, descending: Boolean(req.query.desc), limit, offset },
|
||||
);
|
||||
const orderByField = ORDER_BY_FIELD_MAP[orderBy];
|
||||
const sortDirection = req.query.desc ? 'DESC' : 'ASC';
|
||||
|
||||
const installedApps = (await clients.db.read(
|
||||
`SELECT
|
||||
apps.name,
|
||||
apps.uid,
|
||||
apps.title,
|
||||
apps.description,
|
||||
apps.icon,
|
||||
MIN(perm.dt) AS installed_at
|
||||
FROM apps
|
||||
LEFT JOIN user_to_app_permissions AS perm ON apps.id = perm.app_id
|
||||
WHERE perm.user_id = ?
|
||||
GROUP BY apps.id, apps.name, apps.uid, apps.title, apps.description
|
||||
ORDER BY ${orderByField} ${sortDirection}
|
||||
LIMIT ?
|
||||
OFFSET ?`,
|
||||
[actor.user.id, limit, offset],
|
||||
)) as Array<Record<string, unknown>>;
|
||||
|
||||
const apiBaseUrl = extension.config.api_base_url as string | undefined;
|
||||
res.json(
|
||||
installedApps.map((app) => {
|
||||
// An app with no owner_user_id (null/empty) isn't owned by a Puter
|
||||
// user — it's an "external" app. Derive a flag and don't leak the
|
||||
// raw owner id to the client.
|
||||
const { owner_user_id, ...rest } = app;
|
||||
const external = owner_user_id == null || owner_user_id === '';
|
||||
return {
|
||||
...rest,
|
||||
iconUrl: getAppIconUrl(app, { apiBaseUrl }),
|
||||
// Direct subdomain URL for the client to try before iconUrl.
|
||||
iconCdnUrl: getAppIconCdnUrl(app, extension.config),
|
||||
external,
|
||||
};
|
||||
}),
|
||||
installedApps.map((app) => ({
|
||||
...app,
|
||||
iconUrl: getAppIconUrl(app, { apiBaseUrl }),
|
||||
})),
|
||||
);
|
||||
};
|
||||
|
||||
extension.get(
|
||||
'/installedApps',
|
||||
{
|
||||
subdomain: 'api',
|
||||
requireUserActor: true,
|
||||
allowFullAccessToken: true,
|
||||
rateLimit: {
|
||||
scope: 'installed-apps',
|
||||
limit: 120,
|
||||
window: 60_000,
|
||||
key: 'user',
|
||||
},
|
||||
},
|
||||
{ subdomain: 'api', requireUserActor: true },
|
||||
handleInstalledApps,
|
||||
);
|
||||
+22
-203
@@ -8,7 +8,6 @@ import {
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { makeActor, type Actor } from '../src/backend/core/actor.ts';
|
||||
import { runWithContext } from '../src/backend/core/context.ts';
|
||||
import { PuterServer } from '../src/backend/server.ts';
|
||||
import { setupTestServer } from '../src/backend/testUtil.ts';
|
||||
@@ -49,7 +48,7 @@ beforeAll(async () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
await server.shutdown();
|
||||
});
|
||||
|
||||
const seedUser = async () => {
|
||||
@@ -63,6 +62,15 @@ const seedUser = async () => {
|
||||
};
|
||||
|
||||
describe('metering extension — handleMeteringUsage', () => {
|
||||
it('throws HttpError(401) when no user actor is on the context', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
runWithContext({ actor: undefined }, () =>
|
||||
handleMeteringUsage(makeReq(), res),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it('returns usage details merged with allowanceInfo for an authenticated user', async () => {
|
||||
const user = await seedUser();
|
||||
const { res, captured } = makeRes();
|
||||
@@ -80,83 +88,21 @@ describe('metering extension — handleMeteringUsage', () => {
|
||||
(captured.body as Record<string, unknown>).allowanceInfo,
|
||||
).toBeDefined();
|
||||
});
|
||||
|
||||
describe('usage split across apps', () => {
|
||||
const seedUsage = async () => {
|
||||
const user = await seedUser();
|
||||
const owner = { uuid: user.uuid, id: user.id as number };
|
||||
const metering = server.services.metering;
|
||||
await metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: 'app-mine' } }),
|
||||
'kv:read',
|
||||
1,
|
||||
10,
|
||||
);
|
||||
await metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: 'app-other' } }),
|
||||
'kv:write',
|
||||
1,
|
||||
20,
|
||||
);
|
||||
await metering.incrementUsage(
|
||||
makeActor({ user: owner }),
|
||||
'kv:list',
|
||||
1,
|
||||
40,
|
||||
);
|
||||
await server.stores.meteringBuffer.flushCycle();
|
||||
return owner;
|
||||
};
|
||||
|
||||
const usageAs = async (actor: Actor) => {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () =>
|
||||
handleMeteringUsage(makeReq(), res),
|
||||
);
|
||||
return captured.body as {
|
||||
usage: Record<string, unknown> & { total: number };
|
||||
appTotals: Record<string, unknown>;
|
||||
allowanceInfo: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
|
||||
it('reports only the calling app’s own usage to an app actor', async () => {
|
||||
const owner = await seedUsage();
|
||||
const body = await usageAs(
|
||||
makeActor({ user: owner, app: { uid: 'app-mine' } }),
|
||||
);
|
||||
|
||||
expect(body.usage.total).toBe(10);
|
||||
expect(Object.keys(body.usage).sort()).toEqual([
|
||||
'kv:read',
|
||||
'total',
|
||||
]);
|
||||
expect(body.appTotals).toEqual({
|
||||
'app-mine': { total: 10, count: 1 },
|
||||
});
|
||||
expect(body.allowanceInfo).toMatchObject({
|
||||
remaining: expect.any(Number),
|
||||
monthUsageAllowance: expect.any(Number),
|
||||
});
|
||||
});
|
||||
|
||||
it('still reports the whole account to the user', async () => {
|
||||
const owner = await seedUsage();
|
||||
const body = await usageAs(makeActor({ user: owner }));
|
||||
|
||||
expect(body.usage.total).toBe(70);
|
||||
expect(body.usage).toHaveProperty('kv:read');
|
||||
expect(body.usage).toHaveProperty('kv:write');
|
||||
expect(body.usage).toHaveProperty('kv:list');
|
||||
expect(body.appTotals).toMatchObject({
|
||||
'app-mine': { total: 10, count: 1 },
|
||||
'app-other': { total: 20, count: 1 },
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('metering extension — handleMeteringUsageForApp', () => {
|
||||
it('throws HttpError(401) when no user actor is on the context', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
runWithContext({ actor: undefined }, () =>
|
||||
handleMeteringUsageForApp(
|
||||
makeReq({ params: { appIdOrName: 'any' } }),
|
||||
res,
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it('throws HttpError(400) when no appId is supplied', async () => {
|
||||
const user = await seedUser();
|
||||
const { res } = makeRes();
|
||||
@@ -186,96 +132,6 @@ describe('metering extension — handleMeteringUsageForApp', () => {
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
describe('scoped to the calling app', () => {
|
||||
const appUsageAs = async (actor: Actor, appIdOrName: string) => {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () =>
|
||||
handleMeteringUsageForApp(
|
||||
makeReq({ params: { appIdOrName } }),
|
||||
res,
|
||||
),
|
||||
);
|
||||
return captured.body as { total: number };
|
||||
};
|
||||
|
||||
const seedApps = async () => {
|
||||
const user = await seedUser();
|
||||
const owner = { uuid: user.uuid, id: user.id as number };
|
||||
const mine = `app-${uuidv4()}`;
|
||||
const other = `app-${uuidv4()}`;
|
||||
await server.services.metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: mine } }),
|
||||
'kv:read',
|
||||
1,
|
||||
10,
|
||||
);
|
||||
await server.services.metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: other } }),
|
||||
'kv:write',
|
||||
1,
|
||||
20,
|
||||
);
|
||||
await server.stores.meteringBuffer.flushCycle();
|
||||
return {
|
||||
owner,
|
||||
mine,
|
||||
other,
|
||||
asMine: makeActor({ user: owner, app: { uid: mine } }),
|
||||
};
|
||||
};
|
||||
|
||||
it('lets an app read its own usage', async () => {
|
||||
const { mine, asMine } = await seedApps();
|
||||
expect((await appUsageAs(asMine, mine)).total).toBe(10);
|
||||
});
|
||||
|
||||
it('refuses an app another app’s usage', async () => {
|
||||
const { other, asMine } = await seedApps();
|
||||
await expect(appUsageAs(asMine, other)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses an app another app’s usage by name', async () => {
|
||||
const { owner, other, asMine } = await seedApps();
|
||||
const name = `other-${uuidv4()}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[other, name, name, `https://${name}.example/`, owner.id],
|
||||
);
|
||||
await expect(appUsageAs(asMine, name)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses an app the usage outside any app', async () => {
|
||||
const { asMine } = await seedApps();
|
||||
await expect(appUsageAs(asMine, 'os-global')).rejects.toMatchObject(
|
||||
{ statusCode: 403 },
|
||||
);
|
||||
});
|
||||
|
||||
it('still lets the user read any of their apps', async () => {
|
||||
const { owner, mine, other } = await seedApps();
|
||||
const asUser = makeActor({ user: owner });
|
||||
expect((await appUsageAs(asUser, mine)).total).toBe(10);
|
||||
expect((await appUsageAs(asUser, other)).total).toBe(20);
|
||||
});
|
||||
|
||||
it('lets the user read their usage outside any app', async () => {
|
||||
const { owner } = await seedApps();
|
||||
const asUser = makeActor({ user: owner });
|
||||
await server.services.metering.incrementUsage(
|
||||
asUser,
|
||||
'kv:read',
|
||||
1,
|
||||
5,
|
||||
);
|
||||
await server.stores.meteringBuffer.flushCycle();
|
||||
expect((await appUsageAs(asUser, 'os-global')).total).toBe(5);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('metering extension — handleMeteringGlobalUsage', () => {
|
||||
@@ -309,41 +165,4 @@ describe('metering extension — handleMeteringAllCosts', () => {
|
||||
// per request).
|
||||
expect(costsA).toBe(costsB);
|
||||
});
|
||||
|
||||
it('refreshes a warm catalogue at the UTC price boundary', async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
try {
|
||||
vi.setSystemTime(new Date('2026-11-30T23:59:59Z'));
|
||||
const before = makeRes();
|
||||
await handleMeteringAllCosts(makeReq(), before.res);
|
||||
vi.setSystemTime(new Date('2026-12-01T00:00:00Z'));
|
||||
const after = makeRes();
|
||||
await handleMeteringAllCosts(makeReq(), after.res);
|
||||
expect(
|
||||
(after.captured.body as { costs: unknown[] }).costs,
|
||||
).not.toBe((before.captured.body as { costs: unknown[] }).costs);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
// A scheduled price change has to reach the catalogue without a restart.
|
||||
it('collects the catalogue again once the cache expires', async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
try {
|
||||
vi.setSystemTime(new Date('2026-12-02T00:00:00Z'));
|
||||
const a = makeRes();
|
||||
const b = makeRes();
|
||||
await handleMeteringAllCosts(makeReq(), a.res);
|
||||
vi.setSystemTime(Date.now() + 10 * 60 * 1000 + 1);
|
||||
await handleMeteringAllCosts(makeReq(), b.res);
|
||||
|
||||
const costsA = (a.captured.body as { costs: unknown[] }).costs;
|
||||
const costsB = (b.captured.body as { costs: unknown[] }).costs;
|
||||
expect(costsB).not.toBe(costsA);
|
||||
expect(costsB).toEqual(costsA);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
+29
-157
@@ -1,32 +1,24 @@
|
||||
import type { Request, Response } from 'express';
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { HttpError } from '@heyputer/backend/src/core/http';
|
||||
import {
|
||||
controllersContainers,
|
||||
driversContainers,
|
||||
servicesContainers,
|
||||
} from '@heyputer/backend/src/exports';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import { GLOBAL_APP_KEY } from '@heyputer/backend/src/services/metering/consts';
|
||||
import {
|
||||
creditMultiplierFrom,
|
||||
toCredits,
|
||||
} from '@heyputer/backend/src/services/metering/utils';
|
||||
import type { Request, Response } from 'express';
|
||||
|
||||
const services = extension.import('service');
|
||||
const stores = extension.import('store');
|
||||
const clients = extension.import('client');
|
||||
|
||||
// Catalogues change on deploy or on a scheduled price change, so the cache
|
||||
// expires rather than living as long as the process.
|
||||
const ALL_COSTS_TTL_MS = 10 * 60 * 1000;
|
||||
// Cached on first request — the underlying cost catalogues are baked into
|
||||
// driver/controller source so they only change on deploy.
|
||||
let cachedAllCosts: Record<string, unknown>[] | null = null;
|
||||
let allCostsExpireAt = 0;
|
||||
|
||||
function collectAllCosts(): Record<string, unknown>[] {
|
||||
const all: Record<string, unknown>[] = [];
|
||||
const collect = (
|
||||
source: Record<string, unknown>,
|
||||
kind: 'driver' | 'controller' | 'service',
|
||||
kind: 'driver' | 'controller',
|
||||
) => {
|
||||
for (const [name, instance] of Object.entries(source)) {
|
||||
const fn = (
|
||||
@@ -51,115 +43,44 @@ function collectAllCosts(): Record<string, unknown>[] {
|
||||
};
|
||||
collect(driversContainers as Record<string, unknown>, 'driver');
|
||||
collect(controllersContainers as Record<string, unknown>, 'controller');
|
||||
// Services report the costs that aren't tied to one endpoint — egress,
|
||||
// which is metered for every response there is.
|
||||
collect(servicesContainers as Record<string, unknown>, 'service');
|
||||
return all;
|
||||
}
|
||||
|
||||
// -- Credits scaling ---------------------------------------------------
|
||||
//
|
||||
// When the deployment configures a credit multiplier, every monetary amount
|
||||
// is scaled to display credits BEFORE it leaves the server — raw metered
|
||||
// amounts are never reported. Counts, units, and byte figures pass through
|
||||
// untouched. A response that was scaled says so (`unit: 'credits'`); without
|
||||
// a configured multiplier amounts pass through raw and clients render them
|
||||
// as dollars.
|
||||
|
||||
/** The per-usage-type records of a UsageByType map, costs scaled. */
|
||||
const scaleUsageByType = (
|
||||
usage: Record<string, unknown>,
|
||||
multiplier: number,
|
||||
): Record<string, unknown> => {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(usage)) {
|
||||
// `allowanceUsed` is the month's allowance-charged spend — monetary,
|
||||
// scalar, scaled like the total.
|
||||
if (key === 'total' || key === 'allowanceUsed') {
|
||||
out[key] = toCredits(Number(value) || 0, multiplier);
|
||||
} else if (
|
||||
value &&
|
||||
typeof value === 'object' &&
|
||||
typeof (value as { cost?: unknown }).cost === 'number'
|
||||
) {
|
||||
out[key] = {
|
||||
...(value as Record<string, unknown>),
|
||||
cost: toCredits((value as { cost: number }).cost, multiplier),
|
||||
};
|
||||
} else {
|
||||
out[key] = value;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
export const handleMeteringUsage = async (
|
||||
_req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
// Behind `requireAuth`, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
const actor = Context.get('actor');
|
||||
if (!actor?.user) throw new HttpError(401, 'Authentication required');
|
||||
|
||||
const [actorUsage, allowanceInfo] = await Promise.all([
|
||||
services.metering.getActorCurrentMonthUsageDetails(actor),
|
||||
services.metering.getAllowedUsage(actor),
|
||||
]);
|
||||
const multiplier = creditMultiplierFrom(extension.config);
|
||||
if (!multiplier) {
|
||||
res.json({ ...actorUsage, allowanceInfo });
|
||||
return;
|
||||
}
|
||||
res.json({
|
||||
usage: scaleUsageByType(
|
||||
actorUsage.usage as unknown as Record<string, unknown>,
|
||||
multiplier,
|
||||
),
|
||||
appTotals: Object.fromEntries(
|
||||
Object.entries(actorUsage.appTotals).map(([appId, totals]) => [
|
||||
appId,
|
||||
{ ...totals, total: toCredits(totals.total, multiplier) },
|
||||
]),
|
||||
),
|
||||
allowanceInfo: {
|
||||
...allowanceInfo,
|
||||
remaining: toCredits(allowanceInfo.remaining, multiplier),
|
||||
monthUsageAllowance: toCredits(
|
||||
allowanceInfo.monthUsageAllowance,
|
||||
multiplier,
|
||||
),
|
||||
// Monetary addon fields scale; absent ones stay absent rather
|
||||
// than materializing as NaN/null. purchasedStorage is bytes.
|
||||
addons: Object.fromEntries(
|
||||
Object.entries(allowanceInfo.addons ?? {}).map(([k, v]) => [
|
||||
k,
|
||||
(k === 'purchasedCredits' ||
|
||||
k === 'consumedPurchaseCredits') &&
|
||||
typeof v === 'number'
|
||||
? toCredits(v, multiplier)
|
||||
: v,
|
||||
]),
|
||||
),
|
||||
unit: 'credits',
|
||||
},
|
||||
});
|
||||
res.json({ ...actorUsage, allowanceInfo });
|
||||
};
|
||||
|
||||
export const handleMeteringUsageForApp = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
// Behind `requireAuth`, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
const actor = Context.get('actor');
|
||||
if (!actor?.user) throw new HttpError(401, 'Authentication required');
|
||||
|
||||
let appId = String(req.params.appIdOrName ?? '');
|
||||
if (!appId) throw new HttpError(400, 'appId parameter is required');
|
||||
|
||||
// If not a UUID-shaped app UID or the global sentinel, look up by name.
|
||||
// Which apps an actor may read is MeteringService's call, not this route's.
|
||||
if (!appId.startsWith('app-') && appId !== GLOBAL_APP_KEY) {
|
||||
const app = await stores.app.getByName(appId);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
appId = app.uid;
|
||||
// If not a UUID-shaped app UID, look up by name
|
||||
if (!appId.startsWith('app-')) {
|
||||
const appRows = (await clients.db.read(
|
||||
'SELECT `uid` FROM `apps` WHERE `name` = ? LIMIT 1',
|
||||
[appId],
|
||||
)) as Array<{ uid: string }>;
|
||||
if (appRows.length > 0) {
|
||||
appId = appRows[0].uid;
|
||||
} else {
|
||||
throw new HttpError(404, 'App not found');
|
||||
}
|
||||
}
|
||||
|
||||
const appUsage =
|
||||
@@ -167,18 +88,7 @@ export const handleMeteringUsageForApp = async (
|
||||
actor,
|
||||
appId,
|
||||
);
|
||||
const multiplier = creditMultiplierFrom(extension.config);
|
||||
if (!multiplier) {
|
||||
res.json(appUsage);
|
||||
return;
|
||||
}
|
||||
res.json({
|
||||
...scaleUsageByType(
|
||||
appUsage as unknown as Record<string, unknown>,
|
||||
multiplier,
|
||||
),
|
||||
unit: 'credits',
|
||||
});
|
||||
res.json(appUsage);
|
||||
};
|
||||
|
||||
export const handleMeteringGlobalUsage = async (
|
||||
@@ -189,75 +99,37 @@ export const handleMeteringGlobalUsage = async (
|
||||
res.json(globalUsage);
|
||||
};
|
||||
|
||||
// A hit walks the registries when the in-memory cache is missing or expired.
|
||||
// First hit walks the registries; subsequent hits serve the in-memory cache.
|
||||
export const handleMeteringAllCosts = async (
|
||||
_req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
if (!cachedAllCosts || Date.now() >= allCostsExpireAt) {
|
||||
allCostsExpireAt =
|
||||
(Math.floor(Date.now() / ALL_COSTS_TTL_MS) + 1) * ALL_COSTS_TTL_MS;
|
||||
const raw = collectAllCosts();
|
||||
const multiplier = creditMultiplierFrom(extension.config);
|
||||
// Price the catalogue in the same unit the usage endpoints report:
|
||||
// per-unit credits when a multiplier is configured, raw otherwise.
|
||||
cachedAllCosts = multiplier
|
||||
? raw.map(({ ucentsPerUnit, costs_currency: _cc, ...rest }) =>
|
||||
typeof ucentsPerUnit === 'number'
|
||||
? {
|
||||
...rest,
|
||||
creditsPerUnit: toCredits(
|
||||
ucentsPerUnit,
|
||||
multiplier,
|
||||
),
|
||||
}
|
||||
: rest,
|
||||
)
|
||||
: raw;
|
||||
if (!cachedAllCosts) {
|
||||
cachedAllCosts = collectAllCosts();
|
||||
}
|
||||
res.json({ costs: cachedAllCosts });
|
||||
};
|
||||
|
||||
/** Dashboard reads over the per-actor KV aggregates. */
|
||||
const USAGE_READ_LIMIT = {
|
||||
scope: 'metering-usage',
|
||||
limit: 120,
|
||||
window: 60_000,
|
||||
key: 'user' as const,
|
||||
};
|
||||
|
||||
extension.get(
|
||||
'/metering/usage',
|
||||
{ subdomain: 'api', requireAuth: true, rateLimit: USAGE_READ_LIMIT },
|
||||
{ subdomain: 'api', requireAuth: true },
|
||||
handleMeteringUsage,
|
||||
);
|
||||
|
||||
extension.get(
|
||||
'/metering/usage/:appIdOrName',
|
||||
{ subdomain: 'api', requireAuth: true, rateLimit: USAGE_READ_LIMIT },
|
||||
{ subdomain: 'api', requireAuth: true },
|
||||
handleMeteringUsageForApp,
|
||||
);
|
||||
|
||||
extension.get(
|
||||
'/metering/globalUsage',
|
||||
{
|
||||
subdomain: 'api',
|
||||
adminOnly: true,
|
||||
// Sums across every shard of the global aggregate. Admin-gated, so
|
||||
// this is loop protection — but one accidental poll is an
|
||||
// expensive minute.
|
||||
rateLimit: {
|
||||
scope: 'metering-global-usage',
|
||||
limit: 10,
|
||||
window: 60_000,
|
||||
key: 'user',
|
||||
},
|
||||
},
|
||||
{ subdomain: 'api', adminOnly: true },
|
||||
handleMeteringGlobalUsage,
|
||||
);
|
||||
|
||||
extension.get(
|
||||
'/metering/allCosts',
|
||||
{ subdomain: 'api', requireAuth: true, rateLimit: USAGE_READ_LIMIT },
|
||||
{ subdomain: 'api', requireAuth: true },
|
||||
handleMeteringAllCosts,
|
||||
);
|
||||
@@ -71,7 +71,9 @@ describe('serverInfo extension — handleServerInfo', () => {
|
||||
const statfsSpy = vi
|
||||
.spyOn(fs.default, 'statfs')
|
||||
.mockRejectedValue(new Error('statfs unsupported'));
|
||||
const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
const errSpy = vi
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => {});
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await handleServerInfo({} as Request, res);
|
||||
@@ -86,18 +88,4 @@ describe('serverInfo extension — handleServerInfo', () => {
|
||||
statfsSpy.mockRestore();
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('reports an unknown CPU model when os.cpus() comes back empty', async () => {
|
||||
const os = await import('node:os');
|
||||
const cpusSpy = vi.spyOn(os.default, 'cpus').mockReturnValue([]);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await handleServerInfo({} as Request, res);
|
||||
|
||||
const cpu = captured.body!.cpu as Record<string, unknown>;
|
||||
expect(cpu.model).toBe('Unknown');
|
||||
expect(cpu.cores).toBe(0);
|
||||
|
||||
cpusSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
+96
-927
File diff suppressed because it is too large.
Load diff
+71
-317
@@ -1,95 +1,18 @@
|
||||
import {
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import type { FSEntryStore } from '@heyputer/backend/src/stores/fs/FSEntryStore';
|
||||
import { isMissingObjectError } from '@heyputer/backend/src/stores/fs/S3ObjectStore';
|
||||
import crypto from 'node:crypto';
|
||||
import type { Readable } from 'node:stream';
|
||||
import sharp from 'sharp';
|
||||
const clients = extension.import('client');
|
||||
const stores = extension.import('store');
|
||||
|
||||
type ThumbnailEntryStore = Pick<FSEntryStore, 'updateEntryThumbnailByUuid'>;
|
||||
|
||||
const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_THUMBNAIL_PIXELS = 64e6;
|
||||
|
||||
// Namespace every object this extension writes. An fs object's key is its
|
||||
// bare fsentry uuid and the default config points `thumbnailStore.name` at
|
||||
// the same bucket as `s3_bucket`, so without a prefix of our own there is no
|
||||
// way to tell a thumbnail we minted from any other object in the deployment.
|
||||
const THUMBNAIL_KEY_PREFIX = 'thumbnails/';
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
const isUuid = (value: unknown): value is string =>
|
||||
typeof value === 'string' && UUID_PATTERN.test(value);
|
||||
|
||||
// `thumbnails/<entry uuid>/<random uuid>`: the entry segment binds the object
|
||||
// to the entry it was minted for, which is what writes and deletes check.
|
||||
const mintThumbnailKey = (entryUuid: string): string =>
|
||||
`${THUMBNAIL_KEY_PREFIX}${entryUuid}/${crypto.randomUUID()}`;
|
||||
|
||||
/**
|
||||
* Extract the object key from a stored thumbnail pointer, or null when the
|
||||
* pointer isn't one this extension minted.
|
||||
*
|
||||
* `fsentries.thumbnail` is writable through the FS API, so neither half of the
|
||||
* stored string is trusted: the bucket is discarded (callers always pass their
|
||||
* own) and the key must sit under {@link THUMBNAIL_KEY_PREFIX} as either
|
||||
* `<uuid>` (minted before keys were bound to entries) or `<entry uuid>/<uuid>`.
|
||||
* Honouring an arbitrary key would lend this extension's storage credentials to
|
||||
* whatever object the caller named — in the shared-bucket layout that is every
|
||||
* user's file, since an fs object's key is its fsentry uuid. Bare-uuid keys
|
||||
* from before the prefix fail the check and are treated as absent; they are
|
||||
* indistinguishable from a planted pointer, so there is nothing safer to do
|
||||
* with them than stop signing them.
|
||||
*/
|
||||
const resolveThumbnailKey = (pointer: string): string | null => {
|
||||
let key: string;
|
||||
if (pointer.startsWith('s3://')) {
|
||||
const rest = pointer.slice('s3://'.length);
|
||||
const slash = rest.indexOf('/');
|
||||
if (slash === -1) return null;
|
||||
key = rest.slice(slash + 1);
|
||||
} else {
|
||||
let pathname: string;
|
||||
try {
|
||||
pathname = new URL(pointer).pathname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const segments = pathname.replace(/^\/+/, '').split('/');
|
||||
segments.shift(); // bucket
|
||||
key = segments.join('/');
|
||||
}
|
||||
if (!key.startsWith(THUMBNAIL_KEY_PREFIX)) return null;
|
||||
const segments = key.slice(THUMBNAIL_KEY_PREFIX.length).split('/');
|
||||
return segments.length <= 2 && segments.every(isUuid) ? key : null;
|
||||
};
|
||||
|
||||
// The entry a resolved key was minted for. Null for an unbound key, whose
|
||||
// holder can't be told apart from a row that copied or planted the pointer.
|
||||
const thumbnailKeyOwner = (key: string): string | null => {
|
||||
const segments = key.slice(THUMBNAIL_KEY_PREFIX.length).split('/');
|
||||
return segments.length === 2 ? segments[0] : null;
|
||||
};
|
||||
|
||||
// Whether a stored thumbnail names an object in our storage. Anything else is
|
||||
// an external image URL the client supplied: shown as given, never copied or
|
||||
// deleted.
|
||||
const isStoragePointer = (pointer: string, bucketEndpoint: string): boolean =>
|
||||
pointer.startsWith('s3://') ||
|
||||
// Legacy format — remove after full migration
|
||||
(pointer.startsWith('https') &&
|
||||
pointer.includes(new URL(bucketEndpoint).hostname));
|
||||
|
||||
// S3 client + bucket config — lazily resolved after boot from config.
|
||||
let s3Client: S3Client | null = null;
|
||||
let s3PresignClient: S3Client | null = null;
|
||||
@@ -139,6 +62,13 @@ function getPresignClient(): S3Client {
|
||||
return resolveClients().presign;
|
||||
}
|
||||
|
||||
function base64ParseDataUrl(dataURL: string) {
|
||||
dataURL = dataURL.slice(5);
|
||||
const mimeType = dataURL.split(';')[0];
|
||||
const data = Buffer.from(dataURL.split(',')[1], 'base64');
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
// Strictly decode a data: URL and validate the decoded image. Encoded-string
|
||||
// length lies about decoded byte count (whitespace, padding) and says nothing
|
||||
// about pixel count — a 2MB PNG can decompress to hundreds of MB of raster.
|
||||
@@ -165,73 +95,16 @@ async function decodeAndValidateThumbnail(
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
// A storage pointer a client hands back after a presigned upload is kept only
|
||||
// if it names a key minted for this entry and an object within the size bound.
|
||||
async function vetUploadedThumbnail(
|
||||
pointer: string,
|
||||
entryUuid: unknown,
|
||||
deps: { s3: S3Client; bucketName: string },
|
||||
): Promise<string | null> {
|
||||
const key = resolveThumbnailKey(pointer);
|
||||
if (!key || thumbnailKeyOwner(key) !== entryUuid) return null;
|
||||
|
||||
let size: number | undefined;
|
||||
try {
|
||||
const head = await deps.s3.send(
|
||||
new HeadObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
);
|
||||
size = head.ContentLength;
|
||||
} catch (err) {
|
||||
if (!isMissingObjectError(err)) {
|
||||
console.warn(
|
||||
'[thumbnails] failed to check uploaded thumbnail',
|
||||
err,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (typeof size !== 'number') return null;
|
||||
if (size > MAX_THUMBNAIL_BYTES) {
|
||||
// Only reachable on a store that doesn't enforce the signed length.
|
||||
// The key is this entry's own, so removing it is safe.
|
||||
try {
|
||||
await deps.s3.send(
|
||||
new DeleteObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[thumbnails] failed to remove oversized thumbnail',
|
||||
err,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return `s3://${deps.bucketName}/${key}`;
|
||||
}
|
||||
|
||||
// -- thumbnail.created -----------------------------------------------
|
||||
// ── thumbnail.created ───────────────────────────────────────────────
|
||||
// Intercept data-URL thumbnails before they hit the DB: upload to S3
|
||||
// and replace the URL with an s3:// pointer bound to the entry.
|
||||
// and replace the URL with an s3:// pointer.
|
||||
|
||||
export async function handleThumbnailCreated(
|
||||
event: Record<string, unknown>,
|
||||
deps: { s3: S3Client; bucketName: string; bucketEndpoint: string },
|
||||
deps: { s3: S3Client; bucketName: string },
|
||||
): Promise<void> {
|
||||
const url = event.url;
|
||||
if (typeof url !== 'string') return;
|
||||
|
||||
if (!url.startsWith('data:')) {
|
||||
if (isStoragePointer(url, deps.bucketEndpoint)) {
|
||||
event.url = await vetUploadedThumbnail(url, event.uuid, deps);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const entryUuid = event.uuid;
|
||||
if (!isUuid(entryUuid)) {
|
||||
event.url = null;
|
||||
return;
|
||||
}
|
||||
if (typeof url !== 'string' || !url.startsWith('data:')) return;
|
||||
|
||||
const decoded = await decodeAndValidateThumbnail(url);
|
||||
if (!decoded) {
|
||||
@@ -239,7 +112,7 @@ export async function handleThumbnailCreated(
|
||||
return;
|
||||
}
|
||||
|
||||
const key = mintThumbnailKey(entryUuid);
|
||||
const key = crypto.randomUUID();
|
||||
event.url = `s3://${deps.bucketName}/${key}`;
|
||||
|
||||
await deps.s3.send(
|
||||
@@ -268,24 +141,21 @@ export const handleThumbnailUploadPrepare = async (
|
||||
typeof item.contentType === 'string' ? item.contentType.trim() : '';
|
||||
if (!contentType) continue;
|
||||
|
||||
const size = item.size;
|
||||
if (
|
||||
typeof size !== 'number' ||
|
||||
!Number.isInteger(size) ||
|
||||
size < 0 ||
|
||||
size > MAX_THUMBNAIL_BYTES
|
||||
)
|
||||
continue;
|
||||
if (!isUuid(item.item_uid)) continue;
|
||||
if (item.size !== undefined) {
|
||||
const size = Number(item.size);
|
||||
if (
|
||||
!Number.isFinite(size) ||
|
||||
size < 0 ||
|
||||
size > MAX_THUMBNAIL_BYTES
|
||||
)
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = mintThumbnailKey(item.item_uid);
|
||||
const key = crypto.randomUUID();
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: key,
|
||||
ContentType: contentType,
|
||||
// Signing covers `content-length`, so a body of any other size
|
||||
// fails the signature instead of landing.
|
||||
ContentLength: size,
|
||||
});
|
||||
item.uploadUrl = await getSignedUrl(presignClient, command, {
|
||||
expiresIn: 900,
|
||||
@@ -301,172 +171,76 @@ export const handleThumbnailRead = async (
|
||||
s3Presign: S3Client;
|
||||
bucketName: string;
|
||||
bucketEndpoint: string;
|
||||
fsEntry: ThumbnailEntryStore;
|
||||
db: { write: (sql: string, params: unknown[]) => Promise<unknown> };
|
||||
},
|
||||
): Promise<void> => {
|
||||
const thumb = entry.thumbnail;
|
||||
if (typeof thumb !== 'string' || !thumb) return;
|
||||
const presignClient = deps.s3Presign;
|
||||
const signingDate = new Date(
|
||||
Math.floor(Date.now() / 3_600_000) * 3_600_000,
|
||||
);
|
||||
|
||||
if (isStoragePointer(thumb, deps.bucketEndpoint)) {
|
||||
const key = resolveThumbnailKey(thumb);
|
||||
if (!key) {
|
||||
// Not a pointer we minted — refuse to sign it rather than hand
|
||||
// out a presigned read of whatever object it names.
|
||||
entry.thumbnail = null;
|
||||
return;
|
||||
}
|
||||
if (thumb.startsWith('s3://')) {
|
||||
const [bucket, key] = thumb.slice(5).split('/');
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
{ expiresIn: 604800, signingDate },
|
||||
new GetObjectCommand({ Bucket: bucket, Key: key }),
|
||||
{ expiresIn: 604800 },
|
||||
);
|
||||
} else if (
|
||||
thumb.startsWith('https') &&
|
||||
thumb.includes(new URL(deps.bucketEndpoint).hostname)
|
||||
) {
|
||||
// Legacy format — remove after full migration
|
||||
const [bucket, key] = new URL(thumb).pathname.slice(1).split('/');
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: bucket, Key: key }),
|
||||
{ expiresIn: 604800 },
|
||||
);
|
||||
} else if (thumb.startsWith('data')) {
|
||||
// Move an inline thumbnail into storage. The row only takes the new
|
||||
// pointer while it still holds this data URL, so of concurrent reads
|
||||
// one wins and the rest discard their upload.
|
||||
const uuid = entry.uuid ?? entry.uid;
|
||||
if (!isUuid(uuid)) return;
|
||||
// Same bar as an upload; anything else stays inline, unmigrated.
|
||||
const decoded = await decodeAndValidateThumbnail(thumb);
|
||||
if (!decoded) return;
|
||||
const key = mintThumbnailKey(uuid);
|
||||
// Inline data-URL migration: upload to S3 and update the DB entry.
|
||||
const key = crypto.randomUUID();
|
||||
const { mimeType, data } = base64ParseDataUrl(thumb);
|
||||
const newUrl = `s3://${deps.bucketName}/${key}`;
|
||||
|
||||
await deps.s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: key,
|
||||
Body: decoded.data,
|
||||
ContentType: decoded.mimeType,
|
||||
Body: data,
|
||||
ContentType: mimeType,
|
||||
}),
|
||||
);
|
||||
|
||||
let migrated: boolean;
|
||||
try {
|
||||
migrated = await deps.fsEntry.updateEntryThumbnailByUuid(
|
||||
uuid,
|
||||
`s3://${deps.bucketName}/${key}`,
|
||||
thumb,
|
||||
);
|
||||
} catch (err) {
|
||||
// The row may or may not point at the object now, so keep it.
|
||||
console.warn('[thumbnails] inline migration failed', err);
|
||||
return;
|
||||
}
|
||||
if (!migrated) {
|
||||
try {
|
||||
await deps.s3.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: key,
|
||||
}),
|
||||
// Best-effort async DB update
|
||||
const uuid = entry.uuid ?? entry.uid;
|
||||
if (uuid) {
|
||||
deps.db
|
||||
.write(
|
||||
'UPDATE `fsentries` SET `thumbnail` = ? WHERE `uuid` = ?',
|
||||
[newUrl, uuid],
|
||||
)
|
||||
.catch((err: unknown) =>
|
||||
console.warn('[thumbnails] inline migration failed', err),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[thumbnails] failed to remove unused thumbnail',
|
||||
err,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
presignClient,
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
{ expiresIn: 604800, signingDate },
|
||||
{ expiresIn: 604800 },
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const handleFsCopyNodeThumbnail = async (
|
||||
payload: { copy?: { thumbnail?: string | null; uuid?: string } | null },
|
||||
deps: {
|
||||
s3: S3Client;
|
||||
bucketName: string;
|
||||
bucketEndpoint: string;
|
||||
fsEntry: ThumbnailEntryStore;
|
||||
},
|
||||
): Promise<void> => {
|
||||
const copy = payload.copy;
|
||||
const thumbnailUrl = copy?.thumbnail;
|
||||
if (!copy || !isUuid(copy.uuid) || typeof thumbnailUrl !== 'string') return;
|
||||
if (!isStoragePointer(thumbnailUrl, deps.bucketEndpoint)) return;
|
||||
|
||||
// Same trust rule as the read and remove paths: only touch objects this
|
||||
// extension minted.
|
||||
const sourceKey = resolveThumbnailKey(thumbnailUrl);
|
||||
if (!sourceKey) return;
|
||||
|
||||
// The copied row points at the SAME object as its source. Give the copy
|
||||
// an object of its own, bound to it, so removing either entry leaves the
|
||||
// other's thumbnail alone.
|
||||
const newKey = mintThumbnailKey(copy.uuid);
|
||||
try {
|
||||
// Nothing larger than an upload may produce gets duplicated; such an
|
||||
// object came in through an unbounded upload URL.
|
||||
const head = await deps.s3.send(
|
||||
new HeadObjectCommand({ Bucket: deps.bucketName, Key: sourceKey }),
|
||||
);
|
||||
if (
|
||||
typeof head.ContentLength !== 'number' ||
|
||||
head.ContentLength > MAX_THUMBNAIL_BYTES
|
||||
) {
|
||||
throw new Error('thumbnail exceeds the size bound');
|
||||
}
|
||||
// Read and rewrite rather than CopyObject: a copy source skips any path
|
||||
// in the client's endpoint, so it can miss an object Bucket/Key reach.
|
||||
const source = await deps.s3.send(
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: sourceKey }),
|
||||
);
|
||||
// The object can be replaced after the HEAD; bound what gets buffered.
|
||||
if ((source.ContentLength ?? 0) > MAX_THUMBNAIL_BYTES) {
|
||||
(source.Body as Readable | undefined)?.destroy();
|
||||
throw new Error('thumbnail exceeds the size bound');
|
||||
}
|
||||
const body = await source.Body?.transformToByteArray();
|
||||
if (!body) throw new Error('thumbnail has no body');
|
||||
await deps.s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: newKey,
|
||||
Body: body,
|
||||
ContentType: source.ContentType,
|
||||
}),
|
||||
);
|
||||
} catch (err) {
|
||||
// The shared object is gone or oversized — drop the pointer rather
|
||||
// than leave the row advertising a thumbnail it doesn't have.
|
||||
await deps.fsEntry.updateEntryThumbnailByUuid(copy.uuid, null);
|
||||
console.warn('[thumbnails] failed to duplicate thumbnail on copy', err);
|
||||
return;
|
||||
}
|
||||
|
||||
await deps.fsEntry.updateEntryThumbnailByUuid(
|
||||
copy.uuid,
|
||||
`s3://${deps.bucketName}/${newKey}`,
|
||||
);
|
||||
};
|
||||
|
||||
export const handleFsRemoveNodeThumbnail = async (
|
||||
payload: { target: { thumbnail?: string | null; uuid?: string } },
|
||||
deps: { s3: S3Client; bucketName: string; bucketEndpoint: string },
|
||||
payload: { target: Record<string, unknown> },
|
||||
deps: { s3: S3Client },
|
||||
): Promise<void> => {
|
||||
const { thumbnail: thumbnailUrl, uuid } = payload.target;
|
||||
if (!thumbnailUrl || !isStoragePointer(thumbnailUrl, deps.bucketEndpoint))
|
||||
return;
|
||||
const thumbnailUrl = payload.target.thumbnail as string | undefined;
|
||||
if (!thumbnailUrl || !thumbnailUrl.startsWith('s3://')) return;
|
||||
|
||||
// The pointer decides which object gets deleted, and a key is visible to
|
||||
// anyone who can list the entry holding it. Only a key minted for this
|
||||
// entry is deleted; an unbound one may be shared or planted, so it stays.
|
||||
const key = resolveThumbnailKey(thumbnailUrl);
|
||||
if (!key || thumbnailKeyOwner(key) !== uuid) return;
|
||||
|
||||
await deps.s3.send(
|
||||
new DeleteObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
);
|
||||
const [bucket, key] = thumbnailUrl.slice(5).split('/');
|
||||
await deps.s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key }));
|
||||
};
|
||||
|
||||
extension.on(
|
||||
@@ -475,12 +249,11 @@ extension.on(
|
||||
await handleThumbnailCreated(event, {
|
||||
s3: getClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// -- thumbnail.upload.prepare ----------------------------------------
|
||||
// ── thumbnail.upload.prepare ────────────────────────────────────────
|
||||
// Generate pre-signed upload URLs so the client can PUT directly to S3.
|
||||
|
||||
extension.on(
|
||||
@@ -493,7 +266,7 @@ extension.on(
|
||||
},
|
||||
);
|
||||
|
||||
// -- thumbnail.read --------------------------------------------------
|
||||
// ── thumbnail.read ──────────────────────────────────────────────────
|
||||
// Convert s3:// or legacy https:// thumbnails to signed URLs.
|
||||
|
||||
extension.on('thumbnail.read', async (_key, entry: Record<string, unknown>) => {
|
||||
@@ -502,35 +275,16 @@ extension.on('thumbnail.read', async (_key, entry: Record<string, unknown>) => {
|
||||
s3Presign: getPresignClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
fsEntry: stores.fsEntry,
|
||||
db: clients.db,
|
||||
});
|
||||
});
|
||||
|
||||
// -- fs.copy.node ----------------------------------------------------
|
||||
// A copied entry initially shares its source's thumbnail object; duplicate
|
||||
// it so removing either entry can't break the other's thumbnail.
|
||||
|
||||
extension.on('fs.copy.node', async (_key, payload) => {
|
||||
await handleFsCopyNodeThumbnail(
|
||||
payload as {
|
||||
copy?: { thumbnail?: string | null; uuid?: string } | null;
|
||||
},
|
||||
{
|
||||
s3: getClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
fsEntry: stores.fsEntry,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// -- fs.remove.node --------------------------------------------------
|
||||
// ── fs.remove.node ──────────────────────────────────────────────────
|
||||
// Delete S3 thumbnail when the file is removed.
|
||||
|
||||
extension.on('fs.remove.node', async (_key, payload) => {
|
||||
await handleFsRemoveNodeThumbnail(payload, {
|
||||
s3: getClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
});
|
||||
});
|
||||
extension.on(
|
||||
'fs.remove.node',
|
||||
async (_key, payload: { target: Record<string, unknown> }) => {
|
||||
await handleFsRemoveNodeThumbnail(payload, { s3: getClient() });
|
||||
},
|
||||
);
|
||||
@@ -1,355 +0,0 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { Readable } from 'node:stream';
|
||||
import type { Request, Response } from 'express';
|
||||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const SECRET = 'ingress-secret';
|
||||
|
||||
// Only `extension` is stubbed; the secret comparison and the routing are the
|
||||
// real code paths, and the fake service/store layer is the boundary.
|
||||
const state = vi.hoisted(() => ({
|
||||
config: { userEmail: { secret: 'ingress-secret' } } as Record<
|
||||
string,
|
||||
unknown
|
||||
>,
|
||||
routes: [] as Array<{
|
||||
path: string;
|
||||
handler: (req: unknown, res: unknown) => unknown;
|
||||
}>,
|
||||
users: new Map<string, Record<string, unknown>>(),
|
||||
writes: [] as Array<{
|
||||
userId: number;
|
||||
path: string;
|
||||
homeRegion?: string;
|
||||
content?: unknown;
|
||||
}>,
|
||||
feedback: [] as Array<{ to: string; from: string | null; raw: Buffer }>,
|
||||
takeFeedback: true,
|
||||
}));
|
||||
|
||||
vi.mock('@heyputer/backend/src/extensions', () => ({
|
||||
extension: {
|
||||
get config() {
|
||||
return state.config;
|
||||
},
|
||||
post(
|
||||
path: string,
|
||||
_opts: unknown,
|
||||
handler: (req: unknown, res: unknown) => unknown,
|
||||
) {
|
||||
state.routes.push({ path, handler });
|
||||
},
|
||||
import(layer: string) {
|
||||
if (layer === 'client') {
|
||||
return {
|
||||
event: {
|
||||
async emitAndWait(
|
||||
key: string,
|
||||
data: {
|
||||
to: string;
|
||||
from: string | null;
|
||||
raw: Buffer;
|
||||
handled: boolean;
|
||||
},
|
||||
) {
|
||||
if (key !== 'email.ingress.feedback') return;
|
||||
state.feedback.push(data);
|
||||
if (state.takeFeedback) data.handled = true;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (layer === 'store') {
|
||||
return {
|
||||
user: {
|
||||
getByUsername: async (username: string) =>
|
||||
state.users.get(username) ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
fs: {
|
||||
async write(
|
||||
userId: number,
|
||||
{
|
||||
fileMetadata,
|
||||
fileContent,
|
||||
}: {
|
||||
fileMetadata: { path: string };
|
||||
fileContent: unknown;
|
||||
},
|
||||
_uploadTracker?: unknown,
|
||||
_storageAllowanceMax?: number,
|
||||
homeRegion?: string,
|
||||
) {
|
||||
state.writes.push({
|
||||
userId,
|
||||
path: fileMetadata.path,
|
||||
homeRegion,
|
||||
content: fileContent,
|
||||
});
|
||||
return { fsEntry: { path: fileMetadata.path } };
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
import './index.js';
|
||||
|
||||
const permanent = (id: number, username: string) => ({
|
||||
id,
|
||||
username,
|
||||
email: `${username}@example.com`,
|
||||
password: '$2b$10$hash',
|
||||
});
|
||||
/** No password and no email is what makes an account temporary. */
|
||||
const temp = (id: number, username: string) => ({
|
||||
id,
|
||||
username,
|
||||
email: null,
|
||||
password: null,
|
||||
});
|
||||
|
||||
const ingress = async (
|
||||
to: string,
|
||||
over: { secret?: string; contentLength?: string | null } = {},
|
||||
) => {
|
||||
const route = state.routes.find((r) => r.path === '/email/ingress');
|
||||
if (!route) throw new Error('route not registered');
|
||||
|
||||
const body = Buffer.from('From: [email protected]\r\n\r\nhello\r\n');
|
||||
const req = Readable.from([body], { objectMode: false }) as unknown as {
|
||||
headers: Record<string, string>;
|
||||
query: Record<string, string>;
|
||||
};
|
||||
// `contentLength: null` sends no length header at all, which is the one
|
||||
// framing the HTTP parser does not bound for us.
|
||||
req.headers =
|
||||
over.contentLength === null
|
||||
? {}
|
||||
: {
|
||||
'content-length': over.contentLength ?? String(body.length),
|
||||
};
|
||||
req.query = {
|
||||
SECRET: over.secret ?? SECRET,
|
||||
to,
|
||||
from: '[email protected]',
|
||||
subject: 'hi',
|
||||
};
|
||||
|
||||
let status = 200;
|
||||
let ended = false;
|
||||
// `end` takes (chunk, encoding) and no callback, the way the compression
|
||||
// middleware leaves it, and emits `finish` — which is what the refusal
|
||||
// paths hang their drain/destroy on. A stub that accepts a callback
|
||||
// instead hides the bug that took the 413 path to production as a 500.
|
||||
const emitter = new EventEmitter();
|
||||
const res = {
|
||||
status(code: number) {
|
||||
status = code;
|
||||
return res;
|
||||
},
|
||||
end(chunk?: unknown, encoding?: BufferEncoding) {
|
||||
if (chunk) Buffer.byteLength(chunk as string, encoding);
|
||||
ended = true;
|
||||
emitter.emit('finish');
|
||||
},
|
||||
once(event: string, listener: () => void) {
|
||||
emitter.once(event, listener);
|
||||
return res;
|
||||
},
|
||||
} as unknown as Response;
|
||||
|
||||
await route.handler(req as unknown as Request, res);
|
||||
return {
|
||||
status,
|
||||
ended,
|
||||
drained: (req as unknown as Readable).readableFlowing,
|
||||
destroyed: (req as unknown as Readable).destroyed,
|
||||
};
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
state.users.clear();
|
||||
state.writes.length = 0;
|
||||
state.feedback.length = 0;
|
||||
state.takeFeedback = true;
|
||||
});
|
||||
|
||||
describe('temporary accounts cannot receive', () => {
|
||||
test('mail for a temp account is accepted and dropped', async () => {
|
||||
state.users.set('tempy', temp(1, 'tempy'));
|
||||
|
||||
const { status, ended, drained } = await ingress('[email protected]');
|
||||
// Accepted, so the relay neither retries nor bounces to the sender.
|
||||
expect(status).toBe(200);
|
||||
expect(ended).toBe(true);
|
||||
// The body is read off the wire rather than left unconsumed.
|
||||
expect(drained).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('mail for a permanent account still lands in the mailbox', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(200);
|
||||
expect(state.writes).toHaveLength(1);
|
||||
expect(state.writes[0].userId).toBe(2);
|
||||
expect(state.writes[0].path).toMatch(/^\/bob\/\.mail\/objects\//);
|
||||
});
|
||||
|
||||
test('an unknown address is still rejected outright', async () => {
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(404);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('inbound mail is placed in the recipient home region', () => {
|
||||
/** A permanent account with explicit placement columns. */
|
||||
const placed = (
|
||||
id: number,
|
||||
username: string,
|
||||
over: { home?: string | null; signup_server?: string | null },
|
||||
) => ({
|
||||
...permanent(id, username),
|
||||
home: null,
|
||||
signup_server: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
test("uses the recipient's own home region when they have one", async () => {
|
||||
state.users.set('bob', placed(2, 'bob', { home: 'frankfurt' }));
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('frankfurt');
|
||||
});
|
||||
|
||||
test('falls back to the server that served their signup', async () => {
|
||||
// Every account that predates the `home` column is this case.
|
||||
state.users.set(
|
||||
'bob',
|
||||
placed(2, 'bob', { home: null, signup_server: 'london' }),
|
||||
);
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('london');
|
||||
});
|
||||
|
||||
test('falls back to the primary region when neither is set', async () => {
|
||||
state.users.set('bob', placed(2, 'bob', {}));
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('oregon');
|
||||
});
|
||||
|
||||
test('home wins over signup_server when both are set', async () => {
|
||||
state.users.set(
|
||||
'bob',
|
||||
placed(2, 'bob', { home: 'sydney', signup_server: 'london' }),
|
||||
);
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('sydney');
|
||||
});
|
||||
});
|
||||
|
||||
describe('refusals dispose of the body nobody read', () => {
|
||||
test('an unknown recipient drains it and keeps the connection', async () => {
|
||||
// The secret checked out and the length is under the cap, so the body
|
||||
// was ours to read: discard it rather than resetting a usable socket.
|
||||
const { status, drained, destroyed } =
|
||||
await ingress('[email protected]');
|
||||
expect(status).toBe(404);
|
||||
expect(drained).toBe(true);
|
||||
expect(destroyed).toBe(false);
|
||||
});
|
||||
|
||||
test('a bad secret closes rather than reading the upload', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
secret: 'not-the-secret',
|
||||
});
|
||||
expect(status).toBe(403);
|
||||
// Nothing authenticated this caller; their bytes are not worth reading.
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('no content-length is refused with 411', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
contentLength: null,
|
||||
});
|
||||
expect(status).toBe(411);
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('a declared length over the cap is refused with 413', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
contentLength: String(25 * 1024 * 1024 + 1),
|
||||
});
|
||||
expect(status).toBe(413);
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('feedback addresses', () => {
|
||||
test('are handed to a listener and reach no mailbox', async () => {
|
||||
const { status, ended } = await ingress('[email protected]');
|
||||
expect(status).toBe(200);
|
||||
expect(ended).toBe(true);
|
||||
expect(state.feedback).toHaveLength(1);
|
||||
expect(state.feedback[0].to).toBe('[email protected]');
|
||||
expect(state.feedback[0].from).toBe('[email protected]');
|
||||
expect(state.feedback[0].raw.toString()).toContain('hello');
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('match the local part case-insensitively', async () => {
|
||||
await ingress('[email protected]');
|
||||
expect(state.feedback).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('fall through to normal routing when nobody takes them', async () => {
|
||||
state.takeFeedback = false;
|
||||
state.users.set('abuse', permanent(3, 'abuse'));
|
||||
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(200);
|
||||
expect(state.feedback).toHaveLength(1);
|
||||
// The body was already read, so the buffered copy is what is stored.
|
||||
expect(state.writes).toHaveLength(1);
|
||||
expect(Buffer.isBuffer(state.writes[0].content)).toBe(true);
|
||||
});
|
||||
|
||||
test('an unclaimed one with no such account is still a 404', async () => {
|
||||
state.takeFeedback = false;
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(404);
|
||||
});
|
||||
|
||||
test('only count on a Puter mail domain', async () => {
|
||||
state.users.set('abuse', permanent(3, 'abuse'));
|
||||
await ingress('[email protected]');
|
||||
expect(state.feedback).toEqual([]);
|
||||
});
|
||||
|
||||
test('are refused over the feedback size cap without being read', async () => {
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
contentLength: String(5 * 1024 * 1024 + 1),
|
||||
});
|
||||
expect(status).toBe(413);
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.feedback).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,161 +0,0 @@
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import type { Request } from 'express';
|
||||
import {
|
||||
MAX_MESSAGE_BYTES,
|
||||
isPuterEmailAddress,
|
||||
isTempUser,
|
||||
puterEmailUsername,
|
||||
storeInboxMessage,
|
||||
} from '@heyputer/backend/src/services/email/mailbox.js';
|
||||
|
||||
// copied form peer secrets checker
|
||||
const COMPARE_KEY = randomBytes(32);
|
||||
const secretsEqual = (a: string, b: string): boolean =>
|
||||
timingSafeEqual(
|
||||
createHmac('sha256', COMPARE_KEY).update(a).digest(),
|
||||
createHmac('sha256', COMPARE_KEY).update(b).digest(),
|
||||
);
|
||||
|
||||
const USER_EMAIL_CONFIG = (extension.config as Record<string, unknown>)
|
||||
.userEmail as { secret?: string; feedbackAddresses?: unknown } | undefined;
|
||||
|
||||
const INGRESS_SECRET = USER_EMAIL_CONFIG?.secret;
|
||||
|
||||
/**
|
||||
* Local parts that take complaint feedback rather than naming a mailbox. The
|
||||
* same names are reserved at signup, so no account can be handed them.
|
||||
*/
|
||||
const DEFAULT_FEEDBACK_ADDRESSES = ['fbl', 'abuse', 'postmaster'];
|
||||
const FEEDBACK_ADDRESSES = new Set(
|
||||
(Array.isArray(USER_EMAIL_CONFIG?.feedbackAddresses)
|
||||
? USER_EMAIL_CONFIG.feedbackAddresses
|
||||
: DEFAULT_FEEDBACK_ADDRESSES
|
||||
).map((local) => String(local).toLowerCase()),
|
||||
);
|
||||
|
||||
/**
|
||||
* Feedback is handed over whole, so it is read into memory; a complaint report
|
||||
* quotes one message and has no business near the mailbox ceiling.
|
||||
*/
|
||||
const MAX_FEEDBACK_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
const isFeedbackAddress = (to: string): boolean =>
|
||||
isPuterEmailAddress(to) &&
|
||||
FEEDBACK_ADDRESSES.has(puterEmailUsername(to).toLowerCase());
|
||||
|
||||
const readBody = async (req: Request): Promise<Buffer> => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) chunks.push(chunk as Buffer);
|
||||
return Buffer.concat(chunks);
|
||||
};
|
||||
|
||||
if (!INGRESS_SECRET) {
|
||||
console.warn(
|
||||
'[useremailrx] config.userEmail.secret unset - email ingress disabled',
|
||||
);
|
||||
} else {
|
||||
extension.post(
|
||||
'/email/ingress',
|
||||
{
|
||||
subdomain: 'api',
|
||||
},
|
||||
async (req, res) => {
|
||||
// Answer first, then deal with the body nobody read. Touching the
|
||||
// request before the response has flushed takes the response down
|
||||
// with the connection and the caller sees a reset with no status,
|
||||
// so this hangs off `finish` — and it has to be the event rather
|
||||
// than `res.end(cb)`, because the compression middleware replaces
|
||||
// `res.end` with an `(chunk, encoding)` signature that has no
|
||||
// callback parameter, so a callback lands in `chunk` and dies in
|
||||
// `Buffer.byteLength`.
|
||||
//
|
||||
// `drain` discards the rest of a body we were entitled to read and
|
||||
// leaves the connection reusable. `close` is for bodies we do not
|
||||
// want to read at all: an unauthenticated caller's, or one whose
|
||||
// declared length is missing or over the cap.
|
||||
const refuse = (status: number, how: 'drain' | 'close') => {
|
||||
res.once('finish', () =>
|
||||
how === 'drain' ? req.resume() : req.destroy(),
|
||||
);
|
||||
res.status(status).end();
|
||||
};
|
||||
|
||||
const presented = req.query.SECRET;
|
||||
if (
|
||||
typeof presented !== 'string' ||
|
||||
!secretsEqual(presented, INGRESS_SECRET)
|
||||
) {
|
||||
// Nothing has authenticated this caller, so refuse to spend
|
||||
// bandwidth reading whatever they were uploading.
|
||||
refuse(403, 'close');
|
||||
return;
|
||||
}
|
||||
|
||||
const size = Number(req.headers['content-length']);
|
||||
if (!Number.isInteger(size) || size <= 0) {
|
||||
refuse(411, 'close');
|
||||
return;
|
||||
}
|
||||
if (size > MAX_MESSAGE_BYTES) {
|
||||
refuse(413, 'close');
|
||||
return;
|
||||
}
|
||||
|
||||
const stores = extension.import('store');
|
||||
const services = extension.import('service');
|
||||
|
||||
const to = String(req.query.to ?? '');
|
||||
let content: Buffer | Request = req;
|
||||
if (isFeedbackAddress(to)) {
|
||||
if (size > MAX_FEEDBACK_BYTES) {
|
||||
refuse(413, 'close');
|
||||
return;
|
||||
}
|
||||
const raw = await readBody(req);
|
||||
const event = {
|
||||
to,
|
||||
from:
|
||||
typeof req.query.from === 'string'
|
||||
? req.query.from
|
||||
: null,
|
||||
raw,
|
||||
handled: false,
|
||||
};
|
||||
await extension
|
||||
.import('client')
|
||||
.event.emitAndWait('email.ingress.feedback', event, {});
|
||||
if (event.handled) {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
// Nobody took it: route it like any other message.
|
||||
content = raw;
|
||||
}
|
||||
|
||||
const user = await stores.user.getByUsername(
|
||||
puterEmailUsername(to),
|
||||
);
|
||||
if (!user) {
|
||||
refuse(404, 'drain');
|
||||
return;
|
||||
}
|
||||
|
||||
if (isTempUser(user)) {
|
||||
if (content === req) req.resume();
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
await storeInboxMessage(services.fs, user, {
|
||||
subject: req.query.subject,
|
||||
content,
|
||||
size: Buffer.isBuffer(content) ? content.length : size,
|
||||
});
|
||||
|
||||
const { SECRET: _secret, ..._loggableQuery } = req.query;
|
||||
// console.log('got email: ', puterUser, loggableQuery, size);
|
||||
res.end();
|
||||
},
|
||||
);
|
||||
}
|
||||
Generated
-12
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"name": "emailrx",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "emailrx",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"name": "emailrx",
|
||||
"version": "1.0.0",
|
||||
"main": "index.ts",
|
||||
"description": "Internal service which handles routing puter emails to users"
|
||||
}
|
||||
+17
-460
@@ -1,10 +1,14 @@
|
||||
import type { Request, Response } from 'express';
|
||||
import { installSeatStub } from '@heyputer/backend/testFixtures/seatStub';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { makeActor, type Actor } from '../src/backend/core/actor.ts';
|
||||
import {
|
||||
afterAll,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { runWithContext } from '../src/backend/core/context.ts';
|
||||
import { configContainer } from '../src/backend/exports.ts';
|
||||
import { PuterServer } from '../src/backend/server.ts';
|
||||
import { setupTestServer } from '../src/backend/testUtil.ts';
|
||||
import { handleWhoami } from './whoami.ts';
|
||||
@@ -47,7 +51,7 @@ beforeAll(async () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
await server.shutdown();
|
||||
});
|
||||
|
||||
const seedUser = async () => {
|
||||
@@ -61,78 +65,6 @@ const seedUser = async () => {
|
||||
};
|
||||
|
||||
describe('whoami extension — handleWhoami', () => {
|
||||
// The sidebar label needs this at boot, which is why it rides whoami
|
||||
// rather than a call of its own.
|
||||
describe('the team an account belongs to', () => {
|
||||
// A seat is created, never adopted, so it must have no password.
|
||||
const seedSeat = async () => {
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
return server.stores.user.create({
|
||||
username: `wseat_${slug}`,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: null,
|
||||
});
|
||||
};
|
||||
|
||||
const seatOf = async (teamName: string) => {
|
||||
const seat = await seedSeat();
|
||||
const team = {
|
||||
uid: `wt-${Math.random().toString(36).slice(2, 9)}`,
|
||||
name: teamName,
|
||||
};
|
||||
installSeatStub(server as never, [seat.id as number], {
|
||||
teamUid: team.uid,
|
||||
teamName: team.name,
|
||||
});
|
||||
return { seat, team };
|
||||
};
|
||||
|
||||
it('names no team where teams is off', async () => {
|
||||
const { seat } = await seatOf('Acme Corp');
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: seat.uuid, id: seat.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
// Off, so the seat is never looked up.
|
||||
expect((captured.body as { team?: unknown }).team).toBeUndefined();
|
||||
});
|
||||
|
||||
it('says nothing for an account that is not a seat', async () => {
|
||||
const user = await seedUser();
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
expect(captured.body).not.toHaveProperty('team');
|
||||
});
|
||||
|
||||
it('withholds it from an app actor', async () => {
|
||||
// Same class as the phone number: a seat's employer is not an
|
||||
// app's business.
|
||||
const { seat } = await seatOf('Acme Corp');
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{
|
||||
actor: makeActor({
|
||||
user: { uuid: seat.uuid, id: seat.id as number },
|
||||
app: { uid: 'app-1' },
|
||||
}),
|
||||
},
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
expect(captured.body).not.toHaveProperty('team');
|
||||
});
|
||||
});
|
||||
|
||||
it('returns 401 when no actor is on the context', async () => {
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
@@ -160,81 +92,6 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(captured.body).toEqual({ error: 'User not found' });
|
||||
});
|
||||
|
||||
describe("an app's view of the email", () => {
|
||||
const appActorFor = async (user: { uuid: string; id: unknown }) => {
|
||||
const app = await (
|
||||
server.stores.app.create as unknown as (
|
||||
fields: Record<string, unknown>,
|
||||
opts: { ownerUserId: number },
|
||||
) => Promise<{ uid: string; id: number }>
|
||||
)(
|
||||
{
|
||||
name: `we-${uuidv4()}`,
|
||||
title: 'Email reader',
|
||||
index_url: `https://${uuidv4()}.example/`,
|
||||
},
|
||||
{ ownerUserId: user.id as number },
|
||||
);
|
||||
return {
|
||||
app,
|
||||
actor: makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: app.uid, id: app.id },
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
const whoamiAs = async (actor: Actor) => {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () =>
|
||||
handleWhoami(makeReq(), res),
|
||||
);
|
||||
return captured;
|
||||
};
|
||||
|
||||
it('is withheld until the user grants it, then shown', async () => {
|
||||
const user = await seedUser();
|
||||
const { app, actor } = await appActorFor(user);
|
||||
|
||||
const before = (await whoamiAs(actor)).body as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
expect(before).not.toHaveProperty('email');
|
||||
expect(before).not.toHaveProperty('unconfirmed_email');
|
||||
|
||||
await server.services.permission.grantUserAppPermission(
|
||||
makeActor({ user: { uuid: user.uuid, id: user.id as number } }),
|
||||
app.uid,
|
||||
`user:${user.uuid}:email:read`,
|
||||
);
|
||||
const after = (await whoamiAs(actor)).body as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
expect(after.email).toBe(user.email);
|
||||
expect(after.unconfirmed_email).toBe(user.email);
|
||||
});
|
||||
|
||||
it('answers 404 for a missing account even when the check fails', async () => {
|
||||
const check = vi
|
||||
.spyOn(server.services.permission, 'check')
|
||||
.mockRejectedValue(new Error('scan failed'));
|
||||
try {
|
||||
const captured = await whoamiAs(
|
||||
makeActor({
|
||||
user: { uuid: 'ghost-uuid', id: 99_999_998 },
|
||||
app: { uid: 'app-ghost', id: 1 },
|
||||
}),
|
||||
);
|
||||
expect(captured.statusCode).toBe(404);
|
||||
expect(captured.body).toEqual({ error: 'User not found' });
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('returns full user details for a user actor', async () => {
|
||||
const user = await seedUser();
|
||||
const { res, captured } = makeRes();
|
||||
@@ -250,92 +107,10 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(body.email).toBe(user.email);
|
||||
expect(body.is_temp).toBe(false);
|
||||
expect(body.oidc_only).toBe(false);
|
||||
// is_user_token is present (true) for user actors.
|
||||
expect(body.is_user_token).toBe(true);
|
||||
// Account creation time, in unix seconds.
|
||||
expect(typeof body.created_ts).toBe('number');
|
||||
expect(body.created_ts).toBeGreaterThan(0);
|
||||
// `directories` is only sent to user actors — confirm it’s present.
|
||||
expect(body.directories).toBeDefined();
|
||||
});
|
||||
|
||||
it('leaves out taskbar items saved on the account', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.update(user.id as number, {
|
||||
taskbar_items: JSON.stringify([{ type: 'app', name: 'editor' }]),
|
||||
});
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
expect(captured.body).not.toHaveProperty('taskbar_items');
|
||||
});
|
||||
|
||||
it('leaves out a desktop background saved on the account', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.update(user.id as number, {
|
||||
desktop_bg_url: 'https://cdn.test/wallpaper.png',
|
||||
desktop_bg_color: '#000000',
|
||||
desktop_bg_fit: 'cover',
|
||||
});
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
expect(body).not.toHaveProperty('desktop_bg_url');
|
||||
expect(body).not.toHaveProperty('desktop_bg_color');
|
||||
expect(body).not.toHaveProperty('desktop_bg_fit');
|
||||
});
|
||||
|
||||
it('reports the SMS-to-card fallback only once a send has opened it', async () => {
|
||||
const user = await server.stores.user.create({
|
||||
username: `wuser_${Math.random().toString(36).slice(2, 8)}`,
|
||||
uuid: uuidv4(),
|
||||
password: 'hashedpw',
|
||||
email: `${Math.random().toString(36).slice(2, 8)}@example.com`,
|
||||
requires_phone_verification: true,
|
||||
} as never);
|
||||
const prev = configContainer.phone_verification_card_fallback;
|
||||
configContainer.phone_verification_card_fallback = {
|
||||
enabled: true,
|
||||
} as never;
|
||||
try {
|
||||
const before = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), before.res),
|
||||
);
|
||||
// Phone-gated, but the user has attempts left — no offer yet.
|
||||
expect(
|
||||
before.captured.body as Record<string, unknown>,
|
||||
).toMatchObject({ card_fallback_available: false });
|
||||
|
||||
// Exhausting SMS attempts stamps this flag; whoami is then the only
|
||||
// thing that can still tell a reloading GUI about the offer, since
|
||||
// further sends are rejected by the route's own rate limit.
|
||||
await server.stores.kv.set({
|
||||
key: `card-fallback-open:${user.id}`,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const after = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), after.res),
|
||||
);
|
||||
expect(
|
||||
after.captured.body as Record<string, unknown>,
|
||||
).toMatchObject({ card_fallback_available: true });
|
||||
} finally {
|
||||
configContainer.phone_verification_card_fallback = prev;
|
||||
}
|
||||
// taskbar_items is only sent to user actors.
|
||||
expect(body).toHaveProperty('taskbar_items');
|
||||
});
|
||||
|
||||
it('only forwards allow-listed feature flags', async () => {
|
||||
@@ -355,248 +130,30 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(flags.payment_bypass).toBeUndefined();
|
||||
});
|
||||
|
||||
it('strips created_ts and human_readable_age fields for app actors', async () => {
|
||||
it('strips desktop_bg_* and human_readable_age fields for app actors', async () => {
|
||||
const user = await seedUser();
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
{
|
||||
actor: makeActor({
|
||||
actor: {
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
},
|
||||
},
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
expect(body.app_name).toBe('app-test-actor');
|
||||
// is_user_token is stripped for app actors.
|
||||
expect(body.is_user_token).toBeUndefined();
|
||||
expect(body.desktop_bg_url).toBeUndefined();
|
||||
expect(body.desktop_bg_color).toBeUndefined();
|
||||
expect(body.desktop_bg_fit).toBeUndefined();
|
||||
expect(body.human_readable_age).toBeUndefined();
|
||||
// Account age, in either form, is not exposed to apps.
|
||||
expect(body.created_ts).toBeUndefined();
|
||||
// Directories are user-only.
|
||||
expect(body.directories).toBeUndefined();
|
||||
});
|
||||
|
||||
describe('fields added by whoami.details listeners', () => {
|
||||
const addAccountDetails = (
|
||||
_key: unknown,
|
||||
event: { details: Record<string, unknown> },
|
||||
) => {
|
||||
Object.assign(event.details, {
|
||||
subscribed: true,
|
||||
paid_storage: 1024,
|
||||
subscription: { tier: 'pro', status: 'trialing' },
|
||||
accountEligible: true,
|
||||
});
|
||||
};
|
||||
|
||||
const whoamiWithListener = async (
|
||||
actor: Actor,
|
||||
listener: typeof addAccountDetails = addAccountDetails,
|
||||
) => {
|
||||
const { res, captured } = makeRes();
|
||||
server.clients.event.on('whoami.details', listener);
|
||||
try {
|
||||
await runWithContext({ actor }, () =>
|
||||
handleWhoami(makeReq(), res),
|
||||
);
|
||||
} finally {
|
||||
server.clients.event.off('whoami.details', listener);
|
||||
}
|
||||
return captured.body as Record<string, unknown>;
|
||||
};
|
||||
|
||||
it('reach an app actor only as the documented subscribed / paid_storage', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(body.subscribed).toBe(true);
|
||||
expect(body.paid_storage).toBe(1024);
|
||||
expect(body).not.toHaveProperty('subscription');
|
||||
expect(body).not.toHaveProperty('accountEligible');
|
||||
// Fields the handler built itself still ship.
|
||||
expect(body.username).toBe(user.username);
|
||||
expect(body.app_name).toBe('app-test-actor');
|
||||
});
|
||||
|
||||
it('are withheld from an app reached through an access token it issued', async () => {
|
||||
const user = await seedUser();
|
||||
const owner = { uuid: user.uuid, id: user.id as number };
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
user: owner,
|
||||
accessToken: {
|
||||
uid: 'tok-1',
|
||||
issuer: makeActor({
|
||||
user: owner,
|
||||
app: { uid: 'app-issuer' },
|
||||
}),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(body.subscribed).toBe(true);
|
||||
expect(body).not.toHaveProperty('subscription');
|
||||
expect(body).not.toHaveProperty('accountEligible');
|
||||
});
|
||||
|
||||
it('all reach a user actor', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({ user: { uuid: user.uuid, id: user.id as number } }),
|
||||
);
|
||||
|
||||
expect(body).toMatchObject({
|
||||
subscribed: true,
|
||||
paid_storage: 1024,
|
||||
subscription: { tier: 'pro', status: 'trialing' },
|
||||
accountEligible: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('cannot add directories for an app actor, which core omits', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
(_key, event) => {
|
||||
event.details.directories = { '/injected': 'uuid' };
|
||||
},
|
||||
);
|
||||
|
||||
expect(body).not.toHaveProperty('directories');
|
||||
});
|
||||
});
|
||||
|
||||
it('redacts tmp_password from metadata for user actors', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.updateMetadata(user.id as number, {
|
||||
tmp_password: 'bootstrap-secret',
|
||||
hasDevAccountAccess: true,
|
||||
});
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
const metadata = body.metadata as Record<string, unknown>;
|
||||
expect(metadata.tmp_password).toBeUndefined();
|
||||
// Other metadata keys still reach the user's own client.
|
||||
expect(metadata.hasDevAccountAccess).toBe(true);
|
||||
expect(body.hasDevAccountAccess).toBe(true);
|
||||
});
|
||||
|
||||
it('never sends user metadata to app actors', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.updateMetadata(user.id as number, {
|
||||
tmp_password: 'bootstrap-secret',
|
||||
});
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext(
|
||||
{
|
||||
actor: makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
},
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
expect(body.metadata).toBeUndefined();
|
||||
expect(JSON.stringify(body)).not.toContain('bootstrap-secret');
|
||||
});
|
||||
|
||||
it('never exposes phone, card fingerprint or signup identity', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.update(user.id as number, {
|
||||
phone: '+15551234567',
|
||||
card_fingerprint: 'fp_ABC123',
|
||||
requires_phone_verification: false,
|
||||
requires_card_verification: false,
|
||||
});
|
||||
// Guard against a vacuous assertion below: the columns really do hold
|
||||
// the values we then expect never to see on the wire.
|
||||
const stored = await server.stores.user.getById(user.id as number, {
|
||||
cached: false,
|
||||
force: true,
|
||||
});
|
||||
expect(stored?.phone).toBe('+15551234567');
|
||||
expect(stored?.card_fingerprint).toBe('fp_ABC123');
|
||||
|
||||
for (const actor of [
|
||||
makeActor({ user: { uuid: user.uuid, id: user.id as number } }),
|
||||
makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
]) {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () => handleWhoami(makeReq(), res));
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
expect(body.phone).toBeUndefined();
|
||||
expect(body.card_fingerprint).toBeUndefined();
|
||||
expect(body.password).toBeUndefined();
|
||||
expect(body.otp_secret).toBeUndefined();
|
||||
expect(body.signup_ip).toBeUndefined();
|
||||
// Not just absent as a key — the values must not appear anywhere
|
||||
// in the payload (nested under metadata, …).
|
||||
const serialized = JSON.stringify(body);
|
||||
expect(serialized).not.toContain('5551234567');
|
||||
expect(serialized).not.toContain('fp_ABC123');
|
||||
// The verification flags, which the GUI acts on, still ship.
|
||||
expect(body).toHaveProperty('requires_phone_verification');
|
||||
expect(body).toHaveProperty('requires_card_verification');
|
||||
}
|
||||
});
|
||||
|
||||
it('scrubs sensitive keys added to metadata, and leaves the cached row intact', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.updateMetadata(user.id as number, {
|
||||
tmp_password: 'bootstrap-secret',
|
||||
billing: { card_fingerprint: 'fp_NESTED', tier: 'pro' },
|
||||
});
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext(
|
||||
{ actor: { user: { uuid: user.uuid, id: user.id as number } } },
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
const metadata = body.metadata as Record<string, unknown>;
|
||||
expect(metadata.tmp_password).toBeUndefined();
|
||||
// Nested sensitive keys are removed too; siblings survive.
|
||||
expect(metadata.billing).toEqual({ tier: 'pro' });
|
||||
expect(JSON.stringify(body)).not.toContain('fp_NESTED');
|
||||
|
||||
// The scrub works on a copy — server-side state is untouched.
|
||||
const fresh = await server.stores.user.getById(user.id as number, {
|
||||
cached: false,
|
||||
force: true,
|
||||
});
|
||||
expect(fresh?.metadata?.tmp_password).toBe('bootstrap-secret');
|
||||
expect(
|
||||
(fresh?.metadata?.billing as Record<string, unknown>)
|
||||
?.card_fingerprint,
|
||||
).toBe('fp_NESTED');
|
||||
});
|
||||
|
||||
it('marks the user as oidc_only when password is null', async () => {
|
||||
const slug = Math.random().toString(36).slice(2, 8);
|
||||
const oidcUser = await server.stores.user.create({
|
||||
|
||||
+124
-77
@@ -1,9 +1,6 @@
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import {
|
||||
buildUserDetails,
|
||||
scrubSensitive,
|
||||
} from '@heyputer/backend/src/util/userDetails.js';
|
||||
import { getTaskbarItems } from '@heyputer/backend/src/util/taskbarItems.js';
|
||||
import type { Request, Response } from 'express';
|
||||
import TimeAgo from 'javascript-time-ago';
|
||||
import localeEn from 'javascript-time-ago/locale/en';
|
||||
@@ -17,11 +14,14 @@ const timeago = (() => {
|
||||
return new TimeAgo('en-US');
|
||||
})();
|
||||
|
||||
// What an app actor may receive from `whoami.details` listeners. Anything
|
||||
// else a listener adds describes the account, and only ships to user actors.
|
||||
const APP_VISIBLE_LISTENER_KEYS: ReadonlySet<string> = new Set([
|
||||
'subscribed',
|
||||
'paid_storage',
|
||||
// Allowlist of `config.feature_flags` keys safe to surface via /whoami.
|
||||
// Anything not listed here stays server-side, so internal flags
|
||||
// (payment_bypass, staff_only_*, etc.) cannot leak by accident. Add a
|
||||
// flag here when, and only when, the client actually needs to read it.
|
||||
const CLIENT_VISIBLE_FEATURE_FLAGS: ReadonlySet<string> = new Set([
|
||||
'create_shortcut',
|
||||
'download_directory',
|
||||
'prompt_user_when_navigation_away_from_puter',
|
||||
]);
|
||||
|
||||
export const handleWhoami = async (
|
||||
@@ -34,62 +34,135 @@ export const handleWhoami = async (
|
||||
return;
|
||||
}
|
||||
|
||||
const isUser = !actor.effectiveApp;
|
||||
const canReadEmail = async (uuid: unknown) => {
|
||||
try {
|
||||
return await services.permission.check(
|
||||
actor,
|
||||
`user:${uuid}:email:read`,
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
// Started beside the row read: the actor carries the row's uuid.
|
||||
const emailCheck = isUser ? null : canReadEmail(actor.user.uuid);
|
||||
const isUser = !actor.app;
|
||||
const user = await stores.user.getById(actor.user.id);
|
||||
if (!user) {
|
||||
res.status(404).json({ error: 'User not found' });
|
||||
return;
|
||||
}
|
||||
|
||||
const details = await buildUserDetails(
|
||||
user,
|
||||
{ config: extension.config, clients, stores, services },
|
||||
{ isUser },
|
||||
);
|
||||
details.human_readable_age = user.timestamp
|
||||
? timeago.format(new Date(user.timestamp as string))
|
||||
: null;
|
||||
const oidcOnly = user.password === null;
|
||||
const ALLOWED_ICON_SIZES = new Set([16, 32, 64, 128, 256, 512]);
|
||||
const rawIconSize =
|
||||
typeof req.query?.icon_size === 'string'
|
||||
? Number(req.query.icon_size)
|
||||
: undefined;
|
||||
const iconSize =
|
||||
rawIconSize !== undefined && ALLOWED_ICON_SIZES.has(rawIconSize)
|
||||
? rawIconSize
|
||||
: undefined;
|
||||
const noIcons = !iconSize;
|
||||
|
||||
// Feature flags come from `config.feature_flags`. We only forward keys
|
||||
// listed in CLIENT_VISIBLE_FEATURE_FLAGS so internal flags can't leak.
|
||||
// Non-boolean values (e.g. `"true"` as a string) are coerced so the
|
||||
// client never has to guess.
|
||||
const rawFlags = extension.config.feature_flags ?? {};
|
||||
const feature_flags: Record<string, boolean> = {};
|
||||
for (const [k, v] of Object.entries(rawFlags)) {
|
||||
if (CLIENT_VISIBLE_FEATURE_FLAGS.has(k)) {
|
||||
feature_flags[k] = Boolean(v);
|
||||
}
|
||||
}
|
||||
|
||||
const details: Record<string, unknown> = {
|
||||
username: user.username,
|
||||
uuid: user.uuid,
|
||||
email: user.email,
|
||||
unconfirmed_email: user.email,
|
||||
email_confirmed: user.email_confirmed || user.username === 'admin',
|
||||
requires_email_confirmation: user.requires_email_confirmation,
|
||||
desktop_bg_url: user.desktop_bg_url,
|
||||
desktop_bg_color: user.desktop_bg_color,
|
||||
desktop_bg_fit: user.desktop_bg_fit,
|
||||
is_temp: user.password === null && user.email === null,
|
||||
oidc_only: oidcOnly,
|
||||
taskbar_items: isUser
|
||||
? await getTaskbarItems(
|
||||
user,
|
||||
{
|
||||
clients,
|
||||
stores,
|
||||
services,
|
||||
apiBaseUrl: String(extension.config.api_base_url ?? ''),
|
||||
},
|
||||
{ iconSize, noIcons },
|
||||
)
|
||||
: undefined,
|
||||
otp: !!user.otp_enabled,
|
||||
feature_flags,
|
||||
human_readable_age: user.timestamp
|
||||
? timeago.format(new Date(user.timestamp as string))
|
||||
: null,
|
||||
metadata: user.metadata,
|
||||
hasDevAccountAccess: !!user.metadata?.hasDevAccountAccess,
|
||||
};
|
||||
|
||||
// OIDC revalidate URL for password-less accounts
|
||||
if (oidcOnly) {
|
||||
try {
|
||||
const provider = await services.oidc.getLinkedProviderForUser(
|
||||
user.id as number,
|
||||
);
|
||||
if (provider) {
|
||||
const origin = (extension.config.origin ?? '').replace(
|
||||
/\/$/,
|
||||
'',
|
||||
);
|
||||
details.oidc_revalidate_url = `${origin}/auth/oidc/${provider}/start?flow=revalidate&user_uuid=${encodeURIComponent(user.uuid)}`;
|
||||
}
|
||||
} catch {
|
||||
// OIDC not configured
|
||||
}
|
||||
}
|
||||
|
||||
// Directories — only sent to user actors
|
||||
if (isUser) {
|
||||
const directories: Record<string, unknown> = {};
|
||||
const nameToProp: Record<string, string> = {
|
||||
desktop_uuid: `/${user.username}/Desktop`,
|
||||
appdata_uuid: `/${user.username}/AppData`,
|
||||
documents_uuid: `/${user.username}/Documents`,
|
||||
pictures_uuid: `/${user.username}/Pictures`,
|
||||
videos_uuid: `/${user.username}/Videos`,
|
||||
trash_uuid: `/${user.username}/Trash`,
|
||||
};
|
||||
for (const k in nameToProp) {
|
||||
directories[nameToProp[k]] = user[k];
|
||||
}
|
||||
details.directories = directories;
|
||||
}
|
||||
|
||||
// Last activity
|
||||
if (user.last_activity_ts) {
|
||||
try {
|
||||
details.last_activity_ts = Math.round(
|
||||
new Date(user.last_activity_ts as string).getTime() / 1000,
|
||||
);
|
||||
} catch {
|
||||
/* ignore parse error */
|
||||
}
|
||||
}
|
||||
|
||||
// Strip sensitive fields for app actors
|
||||
if (!isUser) {
|
||||
const emailReadable = await (user.uuid === actor.user.uuid
|
||||
? emailCheck
|
||||
: canReadEmail(user.uuid));
|
||||
if (!emailReadable) {
|
||||
const canReadEmail = await services.permission
|
||||
.check(actor, `user:${user.uuid}:email:read`)
|
||||
.catch(() => false);
|
||||
if (!canReadEmail) {
|
||||
delete details.email;
|
||||
delete details.unconfirmed_email;
|
||||
}
|
||||
delete details.desktop_bg_url;
|
||||
delete details.desktop_bg_color;
|
||||
delete details.desktop_bg_fit;
|
||||
delete details.human_readable_age;
|
||||
delete details.created_ts;
|
||||
delete details.is_user_token;
|
||||
delete details.metadata;
|
||||
// An app has no business reading the code its user earns credit with.
|
||||
delete details.referral_code;
|
||||
}
|
||||
|
||||
const app = actor.effectiveApp;
|
||||
if (app) {
|
||||
details.app_name = app.uid;
|
||||
if (actor.app) {
|
||||
details.app_name = actor.app.uid;
|
||||
}
|
||||
|
||||
// A key core left undefined isn't built, so a listener can't fill it for an app.
|
||||
const builtKeys = isUser
|
||||
? null
|
||||
: new Set(
|
||||
Object.keys(details).filter((key) => details[key] !== undefined),
|
||||
);
|
||||
try {
|
||||
await clients.event.emitAndWait(
|
||||
'whoami.details',
|
||||
@@ -99,47 +172,21 @@ export const handleWhoami = async (
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
if (builtKeys) {
|
||||
for (const key of Object.keys(details)) {
|
||||
if (!builtKeys.has(key) && !APP_VISIBLE_LISTENER_KEYS.has(key)) {
|
||||
delete details[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const subscription = details.subscription as
|
||||
{ offering?: Record<string, unknown> } | undefined;
|
||||
| { offering?: Record<string, unknown> }
|
||||
| undefined;
|
||||
if (subscription?.offering) {
|
||||
delete subscription.offering.group;
|
||||
delete subscription.offering.benefits;
|
||||
delete subscription.offering.price_id;
|
||||
}
|
||||
|
||||
// Last word on what ships, after every listener has had its say.
|
||||
scrubSensitive(details);
|
||||
|
||||
res.json(details);
|
||||
};
|
||||
|
||||
extension.get(
|
||||
'/whoami',
|
||||
{
|
||||
subdomain: 'api',
|
||||
requireAuth: true,
|
||||
allowUnconfirmed: true,
|
||||
// The GUI polls this, and each call fans out to every `whoami`
|
||||
// event listener — so it costs more than the response suggests.
|
||||
//
|
||||
// It is also the call everything else leans on to find out who it is
|
||||
// talking to, so it rides along with unrelated work rather than
|
||||
// arriving at its own pace: the ceiling has to clear whatever the
|
||||
// busiest session is doing, not what a person clicks.
|
||||
rateLimit: {
|
||||
scope: 'whoami',
|
||||
limit: 1_800,
|
||||
window: 60_000,
|
||||
key: 'user',
|
||||
},
|
||||
},
|
||||
{ subdomain: 'api', requireAuth: true, allowUnconfirmed: true },
|
||||
handleWhoami,
|
||||
);
|
||||
+12
-27
@@ -11,7 +11,7 @@
|
||||
# What this does, in order:
|
||||
# 1. Checks that docker (with the compose plugin) exists.
|
||||
# 2. Creates ./puter-selfhosted/ (override with $env:PUTER_DIR).
|
||||
# 3. Downloads docker-compose.yml + caddy/Caddyfile from the OSS repo.
|
||||
# 3. Downloads docker-compose.yml + nginx.conf from the OSS repo.
|
||||
# 4. Generates fresh secrets and writes .env + puter/config/config.json.
|
||||
# 5. Runs `docker compose up -d` and prints how to find the admin password.
|
||||
#
|
||||
@@ -23,7 +23,7 @@
|
||||
# PUTER_DIR install directory (default: ./puter-selfhosted)
|
||||
# PUTER_URL base URL to fetch docker-compose.yml (default: GitHub raw, main branch)
|
||||
# PUTER_DOMAIN domain Puter will serve on (default: puter.localhost)
|
||||
# PUTER_PORT HTTP port for Caddy (default: 80)
|
||||
# PUTER_PORT HTTP port for nginx (default: 80)
|
||||
# PUTER_FORCE set to 1 to overwrite existing .env / config.json
|
||||
|
||||
[CmdletBinding()]
|
||||
@@ -77,10 +77,10 @@ if ($LASTEXITCODE -ne 0) {
|
||||
# ── Step 2: install dir ─────────────────────────────────────────────
|
||||
$null = New-Item -ItemType Directory -Force -Path $PuterDir
|
||||
Set-Location $PuterDir
|
||||
$null = New-Item -ItemType Directory -Force -Path 'puter/config', 'puter/data', 'puter/data/caddy', 'puter/tls'
|
||||
$null = New-Item -ItemType Directory -Force -Path 'puter/config', 'puter/data', 'puter/tls'
|
||||
Write-Log "install dir: $((Get-Location).Path)"
|
||||
|
||||
# ── Step 3: docker-compose.yml + Caddy config ──────────────────────
|
||||
# ── Step 3: docker-compose.yml + nginx config ──────────────────────
|
||||
Write-Log "downloading docker-compose.yml from $PuterUrl"
|
||||
try {
|
||||
Invoke-WebRequest -Uri "$PuterUrl/docker-compose.yml" -OutFile 'docker-compose.yml' -UseBasicParsing
|
||||
@@ -88,19 +88,17 @@ try {
|
||||
Die "could not fetch $PuterUrl/docker-compose.yml — $_"
|
||||
}
|
||||
|
||||
# The Caddyfile is domain-agnostic — it answers on every Host and leaves
|
||||
# the subdomain routing to Puter — so there's nothing to template in.
|
||||
Write-Log "downloading caddy/Caddyfile from $PuterUrl"
|
||||
$null = New-Item -ItemType Directory -Force -Path 'caddy'
|
||||
Write-Log "downloading nginx/nginx.conf from $PuterUrl"
|
||||
$null = New-Item -ItemType Directory -Force -Path 'nginx'
|
||||
# If the path was previously auto-created as a directory by a failed
|
||||
# `compose up`, remove it so we can write the file there.
|
||||
if (Test-Path 'caddy/Caddyfile' -PathType Container) {
|
||||
Remove-Item 'caddy/Caddyfile' -Recurse -Force
|
||||
if (Test-Path 'nginx/nginx.conf' -PathType Container) {
|
||||
Remove-Item 'nginx/nginx.conf' -Recurse -Force
|
||||
}
|
||||
try {
|
||||
Invoke-WebRequest -Uri "$PuterUrl/caddy/Caddyfile" -OutFile 'caddy/Caddyfile' -UseBasicParsing
|
||||
Invoke-WebRequest -Uri "$PuterUrl/nginx/nginx.conf" -OutFile 'nginx/nginx.conf' -UseBasicParsing
|
||||
} catch {
|
||||
Die "could not fetch $PuterUrl/caddy/Caddyfile — $_"
|
||||
Die "could not fetch $PuterUrl/nginx/nginx.conf — $_"
|
||||
}
|
||||
|
||||
# ── Step 4: secrets, .env, config.json ──────────────────────────────
|
||||
@@ -115,16 +113,12 @@ if ($writeConfig) {
|
||||
$mariadbRootPw = New-HexSecret 32
|
||||
$mariadbPw = New-HexSecret 32
|
||||
$s3SecretKey = New-HexSecret 32
|
||||
# Two JWT secrets: $jwtSecret is verify-only for legacy v1 tokens
|
||||
# already in circulation; $jwtSecretV2 signs every new token.
|
||||
$jwtSecret = New-HexSecret 64
|
||||
$jwtSecretV2 = New-HexSecret 64
|
||||
$urlSigSecret = New-HexSecret 64
|
||||
|
||||
$envContent = @"
|
||||
HTTP_PORT=$PuterPort
|
||||
# HTTPS_PORT=443 # uncomment after enabling TLS in caddy/Caddyfile
|
||||
# # (see "Step 3 — TLS" in doc/self-hosting.md)
|
||||
# HTTPS_PORT=443 # uncomment after enabling TLS (see doc/selfhosting/full-stack.md)
|
||||
|
||||
MARIADB_ROOT_PASSWORD=$mariadbRootPw
|
||||
MARIADB_DATABASE=puter
|
||||
@@ -148,8 +142,6 @@ S3_BUCKET=puter-local
|
||||
private_app_hosting_domain = "app.$PuterDomain"
|
||||
private_app_hosting_domain_alt = "dev.$PuterDomain"
|
||||
jwt_secret = $jwtSecret
|
||||
jwt_secret_v2 = $jwtSecretV2
|
||||
allow_v1_tokens = $true
|
||||
url_signature_secret = $urlSigSecret
|
||||
database = [ordered]@{
|
||||
engine = 'mysql'
|
||||
@@ -190,13 +182,6 @@ S3_BUCKET=puter-local
|
||||
providers = [ordered]@{
|
||||
ollama = [ordered]@{ enabled = $false }
|
||||
}
|
||||
# Off because a self-hosted install has no paid plans: every account
|
||||
# is on the free policy, so the plan gates would 402 the surfaces that
|
||||
# declare one (the OpenAI/Anthropic-compatible AI endpoints) for
|
||||
# everybody. Same as install.sh.
|
||||
meteringEnforcement = [ordered]@{
|
||||
subscriptions = $false
|
||||
}
|
||||
trust_proxy = 1
|
||||
}
|
||||
$configJson = $config | ConvertTo-Json -Depth 10
|
||||
@@ -215,4 +200,4 @@ Write-Log " cd $PuterDir; docker compose logs -f puter"
|
||||
Write-Log ''
|
||||
Write-Log "open http://${PuterDomain}:${PuterPort} once the puter container is healthy."
|
||||
Write-Log 'first-boot admin password is logged once — grab it with:'
|
||||
Write-Log " cd $PuterDir; docker compose logs puter | Select-String password"
|
||||
Write-Log " cd $PuterDir; docker compose logs puter | Select-String password"
|
||||
+22
-87
@@ -7,8 +7,7 @@
|
||||
# What this does, in order:
|
||||
# 1. Checks that docker (with the compose plugin), curl, and openssl exist.
|
||||
# 2. Creates ./puter-selfhosted/ (override with PUTER_DIR=...).
|
||||
# 3. Downloads docker-compose.yml + caddy/Caddyfile from the OSS repo
|
||||
# (raw.githubusercontent.com).
|
||||
# 3. Downloads docker-compose.yml from the OSS repo (raw.githubusercontent.com).
|
||||
# 4. Generates fresh secrets and writes .env + puter/config/config.json.
|
||||
# 5. Runs `docker compose up -d` and prints the first-boot admin password.
|
||||
#
|
||||
@@ -17,22 +16,11 @@
|
||||
# the compose file + brings the stack up. Set PUTER_FORCE=1 to overwrite.
|
||||
#
|
||||
# Tunable env vars:
|
||||
# PUTER_DIR install directory (default: ./puter-selfhosted)
|
||||
# PUTER_URL base URL to fetch docker-compose.yml (default: GitHub raw, main branch)
|
||||
# PUTER_DOMAIN domain Puter will serve on (default: puter.localhost)
|
||||
# PUTER_PORT HTTP port for Caddy (default: 80)
|
||||
# PUTER_PROTOCOL public scheme: http | https (default: http)
|
||||
# Set https once TLS is terminating in front of Puter —
|
||||
# either the bundled Caddy with your certs (see "Step 3 —
|
||||
# TLS" in doc/self-hosting.md) or your own proxy (Traefik,
|
||||
# a cloud LB). Puter then builds https:// origins, S3
|
||||
# URLs, and redirects. Leave it http and the installer
|
||||
# serves plain HTTP on PUTER_PORT.
|
||||
# PUTER_TRUST_PROXY number of reverse-proxy hops in front (default: 1)
|
||||
# 1 = the bundled Caddy (or a single external proxy);
|
||||
# 2 = two hops (e.g. Cloudflare → Caddy → Puter).
|
||||
# PUTER_ENV prod | dev (default: prod)
|
||||
# PUTER_FORCE set to 1 to overwrite existing .env / config.json
|
||||
# PUTER_DIR install directory (default: ./puter-selfhosted)
|
||||
# PUTER_URL base URL to fetch docker-compose.yml (default: GitHub raw, main branch)
|
||||
# PUTER_DOMAIN domain Puter will serve on (default: puter.localhost)
|
||||
# PUTER_PORT HTTP port for nginx (default: 80)
|
||||
# PUTER_FORCE set to 1 to overwrite existing .env / config.json
|
||||
|
||||
set -eu
|
||||
|
||||
@@ -40,9 +28,6 @@ PUTER_DIR="${PUTER_DIR:-puter-selfhosted}"
|
||||
PUTER_URL="${PUTER_URL:-https://raw.githubusercontent.com/HeyPuter/puter/main}"
|
||||
PUTER_DOMAIN="${PUTER_DOMAIN:-puter.localhost}"
|
||||
PUTER_PORT="${PUTER_PORT:-80}"
|
||||
PUTER_PROTOCOL="${PUTER_PROTOCOL:-http}"
|
||||
PUTER_TRUST_PROXY="${PUTER_TRUST_PROXY:-1}"
|
||||
PUTER_ENV="${PUTER_ENV:-prod}"
|
||||
PUTER_FORCE="${PUTER_FORCE:-0}"
|
||||
|
||||
log() { printf '\033[1;36m[puter-install]\033[0m %s\n' "$*"; }
|
||||
@@ -53,29 +38,9 @@ need() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
|
||||
}
|
||||
|
||||
check_docker() {
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
echo >&2 ""
|
||||
echo >&2 "[puter-install] missing required command: docker"
|
||||
echo >&2 ""
|
||||
echo >&2 "Docker is required to run Puter self-hosted."
|
||||
echo >&2 ""
|
||||
echo >&2 "Installation instructions:"
|
||||
echo >&2 " - macOS (Apple Silicon & Intel): https://docs.docker.com/desktop/setup/install/mac-install/"
|
||||
echo >&2 " (Docker Desktop is recommended. Alternatives: OrbStack, Colima)"
|
||||
echo >&2 " - Linux: https://docs.docker.com/engine/install/"
|
||||
echo >&2 " - Windows: https://docs.docker.com/desktop/setup/install/windows-install/"
|
||||
echo >&2 ""
|
||||
echo >&2 "After installing Docker, make sure it's running and try again."
|
||||
echo >&2 ""
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Step 1: dependency check ────────────────────────────────────────
|
||||
log "checking dependencies"
|
||||
check_docker
|
||||
need docker
|
||||
need curl
|
||||
need openssl
|
||||
docker compose version >/dev/null 2>&1 \
|
||||
@@ -95,27 +60,25 @@ mkdir -p puter/config puter/data puter/tls
|
||||
# bind-mount roots sidesteps the mismatch without guessing each image's
|
||||
# internal UID. (Docker Desktop on macOS/Windows papers over this with
|
||||
# its VM layer; native Linux docker on Debian/Alpine doesn't.)
|
||||
mkdir -p puter/data/valkey puter/data/mariadb puter/data/dynamo puter/data/s3 puter/data/puter puter/data/caddy
|
||||
chmod 0777 puter/data/valkey puter/data/mariadb puter/data/dynamo puter/data/s3 puter/data/puter puter/data/caddy
|
||||
mkdir -p puter/data/valkey puter/data/mariadb puter/data/dynamo puter/data/s3 puter/data/puter
|
||||
chmod 0777 puter/data/valkey puter/data/mariadb puter/data/dynamo puter/data/s3 puter/data/puter
|
||||
log "install dir: $(pwd)"
|
||||
|
||||
# ── Step 3: docker-compose.yml + Caddy config ──────────────────────
|
||||
# ── Step 3: docker-compose.yml + nginx config ──────────────────────
|
||||
log "downloading docker-compose.yml from $PUTER_URL"
|
||||
curl -fsSL "$PUTER_URL/docker-compose.yml" -o docker-compose.yml \
|
||||
|| die "could not fetch $PUTER_URL/docker-compose.yml"
|
||||
|
||||
# Caddy is mounted as `./caddy/Caddyfile:/etc/caddy/Caddyfile:ro,z` — if
|
||||
# nginx is mounted as `./nginx/nginx.conf:/etc/nginx/nginx.conf:ro` — if
|
||||
# the host file is missing, docker silently creates a directory at that
|
||||
# path and the mount fails with "not a directory" at container start.
|
||||
# The Caddyfile is domain-agnostic — it answers on every Host and leaves
|
||||
# the subdomain routing to Puter — so there's nothing to template in.
|
||||
log "downloading caddy/Caddyfile from $PUTER_URL"
|
||||
mkdir -p caddy
|
||||
log "downloading nginx/nginx.conf from $PUTER_URL"
|
||||
mkdir -p nginx
|
||||
# If the path was previously auto-created as a dir by a failed `compose up`,
|
||||
# remove it so curl can write the file.
|
||||
[ -d caddy/Caddyfile ] && rmdir caddy/Caddyfile 2>/dev/null || true
|
||||
curl -fsSL "$PUTER_URL/caddy/Caddyfile" -o caddy/Caddyfile \
|
||||
|| die "could not fetch $PUTER_URL/caddy/Caddyfile"
|
||||
[ -d nginx/nginx.conf ] && rmdir nginx/nginx.conf 2>/dev/null || true
|
||||
curl -fsSL "$PUTER_URL/nginx/nginx.conf" -o nginx/nginx.conf \
|
||||
|| die "could not fetch $PUTER_URL/nginx/nginx.conf"
|
||||
|
||||
# ── Step 4: secrets, .env, config.json ──────────────────────────────
|
||||
write_config=1
|
||||
@@ -129,17 +92,12 @@ if [ "$write_config" = "1" ]; then
|
||||
MARIADB_ROOT_PASSWORD=$(openssl rand -hex 32)
|
||||
MARIADB_PASSWORD=$(openssl rand -hex 32)
|
||||
S3_SECRET_KEY=$(openssl rand -hex 32)
|
||||
# Two JWT secrets: `jwt_secret` is verify-only for legacy v1 tokens
|
||||
# already in circulation; `jwt_secret_v2` signs every new token.
|
||||
# Both are required at boot (`jwt_secret` only when verifying v1).
|
||||
JWT_SECRET=$(openssl rand -hex 64)
|
||||
JWT_SECRET_V2=$(openssl rand -hex 64)
|
||||
URL_SIGNATURE_SECRET=$(openssl rand -hex 64)
|
||||
|
||||
cat > .env <<EOF
|
||||
HTTP_PORT=$PUTER_PORT
|
||||
# HTTPS_PORT=443 # uncomment after enabling TLS in caddy/Caddyfile
|
||||
# # (see "Step 3 — TLS" in doc/self-hosting.md)
|
||||
# HTTPS_PORT=443 # uncomment after enabling TLS (see doc/selfhosting/full-stack.md)
|
||||
|
||||
MARIADB_ROOT_PASSWORD=$MARIADB_ROOT_PASSWORD
|
||||
MARIADB_DATABASE=puter
|
||||
@@ -151,18 +109,13 @@ S3_SECRET_KEY=$S3_SECRET_KEY
|
||||
S3_BUCKET=puter-local
|
||||
EOF
|
||||
|
||||
# `meteringEnforcement.subscriptions` is off here because a self-hosted
|
||||
# install has no paid plans to be on: every account resolves to the free
|
||||
# policy, so leaving the plan gates armed would 402 the surfaces that
|
||||
# declare one (the OpenAI/Anthropic-compatible AI endpoints) for everybody.
|
||||
# The code default is on — this is a property of the install, not of Puter.
|
||||
log "writing puter/config/config.json"
|
||||
cat > puter/config/config.json <<EOF
|
||||
{
|
||||
"domain": "$PUTER_DOMAIN",
|
||||
"protocol": "$PUTER_PROTOCOL",
|
||||
"protocol": "http",
|
||||
"pub_port": $PUTER_PORT,
|
||||
"env": "$PUTER_ENV",
|
||||
"env": "prod",
|
||||
|
||||
"static_hosting_domain": "site.$PUTER_DOMAIN",
|
||||
"static_hosting_domain_alt": "host.$PUTER_DOMAIN",
|
||||
@@ -170,8 +123,6 @@ EOF
|
||||
"private_app_hosting_domain_alt": "dev.$PUTER_DOMAIN",
|
||||
|
||||
"jwt_secret": "$JWT_SECRET",
|
||||
"jwt_secret_v2": "$JWT_SECRET_V2",
|
||||
"allow_v1_tokens": true,
|
||||
"url_signature_secret": "$URL_SIGNATURE_SECRET",
|
||||
|
||||
"database": {
|
||||
@@ -202,7 +153,7 @@ EOF
|
||||
"s3": {
|
||||
"s3Config": {
|
||||
"endpoint": "http://s3:9000",
|
||||
"publicEndpoint": "$PUTER_PROTOCOL://s3.$PUTER_DOMAIN",
|
||||
"publicEndpoint": "http://s3.$PUTER_DOMAIN",
|
||||
"accessKeyId": "puter",
|
||||
"secretAccessKey": "$S3_SECRET_KEY",
|
||||
"region": "us-east-1",
|
||||
@@ -216,27 +167,11 @@ EOF
|
||||
"ollama": { "enabled": false }
|
||||
},
|
||||
|
||||
"meteringEnforcement": {
|
||||
"subscriptions": false
|
||||
},
|
||||
|
||||
"trust_proxy": $PUTER_TRUST_PROXY
|
||||
"trust_proxy": 1
|
||||
}
|
||||
EOF
|
||||
fi
|
||||
|
||||
# `https` only works if something in front is actually terminating it. The
|
||||
# bundled Caddy does that from ./puter/tls once you enable its `:443` block
|
||||
# (see "Step 3 — TLS" in doc/self-hosting.md); your own edge proxy is fine
|
||||
# too. Neither in place means Puter hands out https:// URLs for an endpoint
|
||||
# that only speaks HTTP — broken logins and mixed content.
|
||||
if [ "$PUTER_PROTOCOL" = "https" ] && [ ! -f puter/tls/fullchain.pem ]; then
|
||||
warn "PUTER_PROTOCOL=https but ./puter/tls/fullchain.pem is missing."
|
||||
warn "The bundled Caddy is still serving plain HTTP on port $PUTER_PORT."
|
||||
warn "Either drop a wildcard cert into ./puter/tls and uncomment the :443"
|
||||
warn "block in caddy/Caddyfile, or terminate TLS in your own proxy."
|
||||
fi
|
||||
|
||||
# ── Step 5: bring it up ─────────────────────────────────────────────
|
||||
log "docker compose up -d"
|
||||
docker compose up -d
|
||||
@@ -246,6 +181,6 @@ log "stack starting. first boot takes ~30s while MariaDB initialises."
|
||||
log "follow puter logs:"
|
||||
log " cd $PUTER_DIR && docker compose logs -f puter"
|
||||
log ""
|
||||
log "open $PUTER_PROTOCOL://$PUTER_DOMAIN:$PUTER_PORT once the puter container is healthy."
|
||||
log "open http://$PUTER_DOMAIN:$PUTER_PORT once the puter container is healthy."
|
||||
log "first-boot admin password is logged once — grab it with:"
|
||||
log " cd $PUTER_DIR && docker compose logs puter | grep password"
|
||||
@@ -0,0 +1,135 @@
|
||||
# Reverse proxy in front of Puter — mirrors what the prod ALB does:
|
||||
# accepts every Host header, forwards to the Puter container, and lets
|
||||
# the Puter app handle subdomain-based routing internally (api.*,
|
||||
# site.*, app.*, etc).
|
||||
#
|
||||
# To enable TLS:
|
||||
# 1. Drop your fullchain.pem + privkey.pem into ./puter/tls/.
|
||||
# 2. Uncomment the 443 server{} block below.
|
||||
# 3. Update server_name to your domain (and wildcard subdomains).
|
||||
|
||||
worker_processes auto;
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
# Required for Puter's WebSocket / socket.io upgrades.
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
# Rough size cap that mirrors prod ALB defaults; tune for your
|
||||
# uploads. Puter chunks large uploads, so 1 GiB per request is plenty.
|
||||
client_max_body_size 1024m;
|
||||
proxy_read_timeout 600s;
|
||||
proxy_send_timeout 600s;
|
||||
proxy_buffering off;
|
||||
server_tokens off;
|
||||
|
||||
upstream puter_backend {
|
||||
server puter:4100;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
upstream s3_backend {
|
||||
# RustFS — see `s3` service in docker-compose.full.yml. Browsers
|
||||
# PUT/GET here for presigned-URL uploads / downloads. Routed via
|
||||
# the `s3.<domain>` subdomain so signature verification works
|
||||
# (Host header preserved end-to-end) and so HTTPS stays clean
|
||||
# (no mixed-content from a port-9000 host publish).
|
||||
server s3:9000;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# ── HTTP (port 80) ─────────────────────────────────────────────
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name ~^s3\.;
|
||||
|
||||
location / {
|
||||
proxy_pass http://s3_backend;
|
||||
proxy_http_version 1.1;
|
||||
# Critical: preserve the original Host so RustFS validates
|
||||
# the request against the same host the URL was signed for.
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
listen [::]:80 default_server;
|
||||
server_name _;
|
||||
|
||||
# Note: when you enable TLS, replace this block with a redirect:
|
||||
# return 301 https://$host$request_uri;
|
||||
location / {
|
||||
proxy_pass http://puter_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_set_header X-Forwarded-Port $server_port;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
}
|
||||
}
|
||||
|
||||
# ── HTTPS (port 443) — uncomment after dropping certs in ./puter/tls/ ─
|
||||
# server {
|
||||
# listen 443 ssl;
|
||||
# listen [::]:443 ssl;
|
||||
# http2 on;
|
||||
# server_name ~^s3\.;
|
||||
#
|
||||
# ssl_certificate /etc/nginx/tls/fullchain.pem;
|
||||
# ssl_certificate_key /etc/nginx/tls/privkey.pem;
|
||||
# ssl_protocols TLSv1.2 TLSv1.3;
|
||||
# ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
# ssl_session_cache shared:SSL:10m;
|
||||
# ssl_session_timeout 10m;
|
||||
#
|
||||
# location / {
|
||||
# proxy_pass http://s3_backend;
|
||||
# proxy_http_version 1.1;
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_set_header X-Real-IP $remote_addr;
|
||||
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
# proxy_set_header X-Forwarded-Proto $scheme;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# server {
|
||||
# listen 443 ssl default_server;
|
||||
# listen [::]:443 ssl default_server;
|
||||
# http2 on;
|
||||
# server_name _;
|
||||
#
|
||||
# ssl_certificate /etc/nginx/tls/fullchain.pem;
|
||||
# ssl_certificate_key /etc/nginx/tls/privkey.pem;
|
||||
# ssl_protocols TLSv1.2 TLSv1.3;
|
||||
# ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
# ssl_session_cache shared:SSL:10m;
|
||||
# ssl_session_timeout 10m;
|
||||
#
|
||||
# location / {
|
||||
# proxy_pass http://puter_backend;
|
||||
# proxy_http_version 1.1;
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_set_header X-Real-IP $remote_addr;
|
||||
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
# proxy_set_header X-Forwarded-Proto $scheme;
|
||||
# proxy_set_header X-Forwarded-Host $host;
|
||||
# proxy_set_header X-Forwarded-Port $server_port;
|
||||
# proxy_set_header Upgrade $http_upgrade;
|
||||
# proxy_set_header Connection $connection_upgrade;
|
||||
# }
|
||||
# }
|
||||
}
|
||||
Generated
+6677
-6615
File diff suppressed because it is too large.
Load diff
+27
-45
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "puter.com",
|
||||
"version": "26.07",
|
||||
"version": "2.5.1",
|
||||
"author": "Puter Technologies Inc.",
|
||||
"license": "AGPL-3.0-only",
|
||||
"description": "Desktop environment in the browser!",
|
||||
@@ -10,14 +10,16 @@
|
||||
"lib": "lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.29.7",
|
||||
"@eslint/js": "^9.35.0",
|
||||
"@playwright/test": "^1.56.1",
|
||||
"@stylistic/eslint-plugin": "^5.3.1",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/mime-types": "^3.0.1",
|
||||
"@typescript-eslint/eslint-plugin": "^8.46.1",
|
||||
"@typescript-eslint/parser": "^8.46.1",
|
||||
"@vitest/coverage-v8": "^4.0.14",
|
||||
"babel-loader": "^10.1.1",
|
||||
"babel-plugin-istanbul": "^8.0.0",
|
||||
"@vitest/ui": "^4.0.14",
|
||||
"chalk": "^4.1.0",
|
||||
"clean-css": "^5.3.2",
|
||||
"dotenv": "^16.4.5",
|
||||
@@ -25,44 +27,39 @@
|
||||
"eslint": "^9.35.0",
|
||||
"eslint-config-prettier": "^10.1.8",
|
||||
"eslint-plugin-prettier": "^5.5.5",
|
||||
"eslint-rule-composer": "^0.3.0",
|
||||
"globals": "^15.15.0",
|
||||
"html-webpack-plugin": "^5.6.0",
|
||||
"husky": "^9.1.7",
|
||||
"istanbul-lib-coverage": "^3.2.2",
|
||||
"istanbul-lib-report": "^3.0.1",
|
||||
"istanbul-reports": "^3.2.0",
|
||||
"license-check-and-add": "^4.0.5",
|
||||
"nodemon": "^3.1.0",
|
||||
"prettier": "^3.8.3",
|
||||
"prettier-plugin-jsdoc": "^1.8.1",
|
||||
"simple-git": "^3.32.3",
|
||||
"typescript": "^5.4.5",
|
||||
"uglify-js": "^3.17.4",
|
||||
"vite-plugin-static-copy": "^3.3.0",
|
||||
"vitest": "^4.1.5",
|
||||
"webpack": "^5.88.2",
|
||||
"webpack-cli": "^5.1.1"
|
||||
"webpack-cli": "^5.1.1",
|
||||
"yaml": "^2.8.1"
|
||||
},
|
||||
"scripts": {
|
||||
"test:backend": "npm run setupExtensions && npm run build:workerLib && vitest run --config src/backend/vitest.config.ts ",
|
||||
"test:backend:postgres": "npm run setupExtensions && npm run build:workerLib && PUTER_TEST_DB_ENGINE=postgres vitest run --config src/backend/vitest.config.ts ",
|
||||
"test:puterjs": "npm run setupExtensions && vitest run --config src/puter-js/tests/api/vitest.config.ts",
|
||||
"test:puterjs:node": "npm run test:puterjs -- src/puter-js/tests/api/runners/node.test.ts",
|
||||
"test:puterjs:browser": "npm run test:puterjs -- src/puter-js/tests/api/runners/browser.test.ts",
|
||||
"test:puterjs:workerd": "npm run test:puterjs -- src/puter-js/tests/api/runners/workerd.test.ts",
|
||||
"test:puterjs:coverage": "npm run build:workerLib:coverage && npm run setupExtensions && rm -rf src/puter-js/coverage && PUTER_COVERAGE=1 vitest run --config src/puter-js/tests/api/vitest.config.ts && node ./tools/puterjsCoverageReport.mjs",
|
||||
"build:workerLib:coverage": "cd src/puter-js && npm run build:coverage && cd ../worker && npm run build",
|
||||
"start:gui": "nodemon --exec \"node dev-server.js\" ",
|
||||
"start": "node ./tools/start.mjs",
|
||||
"start:smtp": "node ./tools/start.mjs --smtp",
|
||||
"test:backend": "vitest run --config src/backend/vitest.config.ts ",
|
||||
"start=gui": "nodemon --exec \"node dev-server.js\" ",
|
||||
"start": "node --enable-source-maps -r ./dist/src/backend/telemetry.js ./dist/src/backend/index.js",
|
||||
"prestart": "npm run build:ts",
|
||||
"dev": "npm start",
|
||||
"build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
|
||||
"build": "npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
|
||||
"build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build",
|
||||
"check-translations": "node tools/check-translations.js",
|
||||
"prepare": "husky",
|
||||
"build:ts": "tsc -p tsconfig.build.json && node ./tools/write-dist-package-json.mjs",
|
||||
"check:puterjs:types": "node tools/checkPuterjsTypes.mjs",
|
||||
"typecheck": "node tools/typecheck.mjs",
|
||||
"typecheck:update": "node tools/typecheck.mjs --update",
|
||||
"setupExtensions": "node ./tools/extensionSetup.mjs"
|
||||
"postinstall": "node ./tools/extensionSetup.mjs"
|
||||
},
|
||||
"workspaces": [
|
||||
"src/*"
|
||||
"src/*",
|
||||
"tools/*",
|
||||
"experiments/js-parse-and-output"
|
||||
],
|
||||
"nodemonConfig": {
|
||||
"ext": "js, json, mjs, jsx, svg, css",
|
||||
@@ -72,31 +69,16 @@
|
||||
]
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^3.0.25",
|
||||
"@aws-sdk/client-s3": "^3.1020.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1028.0",
|
||||
"@heyputer/putility": "^1.0.2",
|
||||
"ai": "^6.0.73",
|
||||
"dedent": "^1.5.3",
|
||||
"javascript-time-ago": "^2.5.11",
|
||||
"libphonenumber-js": "1.13.15",
|
||||
"miniflare": "^4.20260617.1",
|
||||
"open": "^10.1.0",
|
||||
"sharp": "^0.35.5"
|
||||
"open": "^10.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=24.0.0"
|
||||
},
|
||||
"allowScripts": {
|
||||
"@google/genai": false,
|
||||
"bcrypt": true,
|
||||
"better-sqlite3": true,
|
||||
"browser-tabs-lock": true,
|
||||
"classic-level": true,
|
||||
"esbuild": true,
|
||||
"fsevents": true,
|
||||
"protobufjs": true,
|
||||
"sharp": true,
|
||||
"workerd": true
|
||||
},
|
||||
"overrides": {
|
||||
"undici": "^7.30.0"
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import type { PuterServer } from './server.ts';
|
||||
import { createTestUser, setupTestServer } from './testUtil.ts';
|
||||
|
||||
/**
|
||||
* Teams is optional, so core has to work when neither the service nor the store
|
||||
* is registered, which is how this test server boots.
|
||||
*/
|
||||
describe('core without a teams registration', () => {
|
||||
let server: PuterServer;
|
||||
|
||||
const makeUser = async (): Promise<{ id: number; username: string }> => {
|
||||
const username = `nt_${Math.random().toString(36).slice(2, 10)}`;
|
||||
const created = (await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
})) as unknown as { id: number };
|
||||
return { id: created.id, username };
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer({
|
||||
// Without this the allowlist check returns before asking Teams.
|
||||
teams_allowed_email_domains: ['test.local'],
|
||||
} as never);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
it('has no teams in the registry', () => {
|
||||
expect(server.services.team).toBeUndefined();
|
||||
expect(server.stores.team).toBeUndefined();
|
||||
});
|
||||
|
||||
it('refuses to boot when the config asks for teams', async () => {
|
||||
await expect(
|
||||
setupTestServer({ teams_enabled: true } as never),
|
||||
).rejects.toThrow(/teams_enabled/);
|
||||
});
|
||||
|
||||
it('deletes an account without asking what it was charged for', async () => {
|
||||
const user = await makeUser();
|
||||
await expect(
|
||||
server.services.userAccount.cascadeDelete(user.id),
|
||||
).resolves.not.toThrow();
|
||||
expect(await server.stores.user.getById(user.id)).toBeFalsy();
|
||||
});
|
||||
|
||||
it('answers the address-claim question signup asks', async () => {
|
||||
const user = await makeUser();
|
||||
const row = (await server.stores.user.getById(user.id))!;
|
||||
// Nothing is a seat, so the holder is returned as-is.
|
||||
await expect(
|
||||
server.services.signup.signupEmailHolder(row.email as string, {
|
||||
releaseSeat: true,
|
||||
}),
|
||||
).resolves.toMatchObject({ id: user.id });
|
||||
});
|
||||
|
||||
it('lists what is shared with an account, folding in no team shares', async () => {
|
||||
const user = await makeUser();
|
||||
await expect(
|
||||
server.services.share.listSharedWithMe({
|
||||
user: { id: user.id },
|
||||
} as never),
|
||||
).resolves.toMatchObject({ items: [] });
|
||||
});
|
||||
|
||||
it('refuses a share aimed at a team, rather than failing', async () => {
|
||||
// A real home directory, so the refusal is the recipient's and not
|
||||
// the entry's.
|
||||
const username = `nt_${Math.random().toString(36).slice(2, 10)}`;
|
||||
const creds = await createTestUser(server, {
|
||||
username,
|
||||
password: 'Nt-Pw-12345!',
|
||||
});
|
||||
const row = (await server.stores.user.getByUsername(username))!;
|
||||
const user = { id: row.id, username };
|
||||
const actor = { user: row } as never;
|
||||
expect(creds).toBeTruthy();
|
||||
// The recipient cannot exist, so this is a 404 and not a crash.
|
||||
await expect(
|
||||
server.services.share.share(actor, {
|
||||
recipient: { teamHandle: 'nobody' },
|
||||
mode: 'read',
|
||||
path: `/${user.username}`,
|
||||
} as never),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('serves the homepage shell with the teams tab off', async () => {
|
||||
const user = await makeUser();
|
||||
const row = (await server.stores.user.getById(user.id))!;
|
||||
let html = '';
|
||||
const res = { send: (out: string) => (html = out) };
|
||||
await server.services.homepage.send(
|
||||
{
|
||||
req: { headers: {}, query: {}, get: () => undefined },
|
||||
res,
|
||||
actor: { user: row },
|
||||
} as never,
|
||||
{ title: 'Puter' } as never,
|
||||
);
|
||||
const params = JSON.parse(
|
||||
/gui\((\{.*?\})\);/s.exec(html)![1].replaceAll('\\u003c', '<'),
|
||||
);
|
||||
expect(params.teams_ui).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,692 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
ALARM_IDLE_TTL_MS,
|
||||
AlarmClient,
|
||||
MAX_TRACKED_ALARMS,
|
||||
} from './AlarmClient';
|
||||
import type { IConfig, IPagerConfig } from '../../types';
|
||||
import type { AlertPayload } from './types';
|
||||
|
||||
const pdEvent = vi.hoisted(() => vi.fn(async () => ({})));
|
||||
vi.mock('@pagerduty/pdjs', () => ({ event: pdEvent }));
|
||||
|
||||
const makeClient = (pager: IPagerConfig = {}) =>
|
||||
new AlarmClient({ serverId: 'test-node', pager } as unknown as IConfig);
|
||||
|
||||
/** Register a capturing handler in place of a real transport. */
|
||||
const capture = (
|
||||
client: AlarmClient,
|
||||
minSeverity?: 'critical' | 'error' | 'warning' | 'info',
|
||||
maxSeverity?: 'critical' | 'error' | 'warning' | 'info',
|
||||
) => {
|
||||
const seen: AlertPayload[] = [];
|
||||
client.addAlertHandler(
|
||||
async (alert) => {
|
||||
seen.push(alert);
|
||||
},
|
||||
{ name: 'capture', minSeverity, maxSeverity },
|
||||
);
|
||||
return seen;
|
||||
};
|
||||
|
||||
describe('AlarmClient severity routing', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
pdEvent.mockClear();
|
||||
});
|
||||
|
||||
it('defaults to critical when the call site says nothing', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('boom', 'everything is on fire');
|
||||
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(seen[0].severity).toBe('critical');
|
||||
});
|
||||
|
||||
it('honours a configured default severity', () => {
|
||||
const client = makeClient({ defaultSeverity: 'info' });
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('boom', 'everything is on fire');
|
||||
|
||||
expect(seen[0].severity).toBe('info');
|
||||
});
|
||||
|
||||
it('skips handlers whose floor the alarm does not reach', () => {
|
||||
const client = makeClient();
|
||||
const paging = capture(client, 'warning');
|
||||
const chat = capture(client, 'info');
|
||||
|
||||
client.create('rate-limit', 'a user hit a limit', {}, 'info');
|
||||
|
||||
expect(paging).toHaveLength(0);
|
||||
expect(chat).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('skips handlers whose ceiling the alarm exceeds', () => {
|
||||
const client = makeClient();
|
||||
const chat = capture(client, 'info', 'info');
|
||||
|
||||
client.create('rate-limit', 'a user hit a limit', {}, 'info');
|
||||
client.create('outage', 'everything is on fire', {}, 'critical');
|
||||
|
||||
expect(chat.map((alert) => alert.id)).toEqual(['rate-limit']);
|
||||
});
|
||||
|
||||
it('retiers an alarm from config', () => {
|
||||
const client = makeClient({
|
||||
severityOverrides: { 'noisy:*': 'info' },
|
||||
});
|
||||
const paging = capture(client, 'warning');
|
||||
const chat = capture(client, 'info');
|
||||
|
||||
client.create('noisy:thing', 'used to page', {}, 'critical');
|
||||
|
||||
expect(paging).toHaveLength(0);
|
||||
expect(chat[0].severity).toBe('info');
|
||||
});
|
||||
|
||||
it('mutes an alarm from config', () => {
|
||||
const client = makeClient({
|
||||
severityOverrides: { 'noisy:thing': 'mute' },
|
||||
});
|
||||
const chat = capture(client, 'info');
|
||||
|
||||
client.create('noisy:thing', 'not worth reporting');
|
||||
client.create('noisy:thing', 'still not worth reporting');
|
||||
|
||||
expect(chat).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('lets config override a known-error rule', () => {
|
||||
const client = makeClient({
|
||||
severityOverrides: { 'known:thing': 'critical' },
|
||||
});
|
||||
client.setKnownErrors([
|
||||
{
|
||||
match: { id: 'known:thing' },
|
||||
action: { type: 'severity', value: 'info' },
|
||||
},
|
||||
]);
|
||||
const paging = capture(client, 'warning');
|
||||
|
||||
client.create('known:thing', 'known but escalated');
|
||||
|
||||
expect(paging[0].severity).toBe('critical');
|
||||
});
|
||||
|
||||
it('still respects a no-alert known-error rule', () => {
|
||||
const client = makeClient();
|
||||
client.setKnownErrors([
|
||||
{ match: { id: 'known:quiet' }, action: { type: 'no-alert' } },
|
||||
]);
|
||||
const chat = capture(client, 'info');
|
||||
|
||||
client.create('known:quiet', 'suppressed');
|
||||
|
||||
expect(chat).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient alert payload', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
it('reports occurrence counts across repeats', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('flap', 'first');
|
||||
client.create('flap', 'second');
|
||||
|
||||
expect(seen[0]).toMatchObject({ repeatCount: 1, isRepeat: false });
|
||||
expect(seen[1]).toMatchObject({ repeatCount: 2, isRepeat: true });
|
||||
});
|
||||
|
||||
it('renders fields as strings and lifts the stack out of the error', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
const error = new Error('kaboom');
|
||||
|
||||
client.create('boom', 'failed', { error, status: 500 }, 'critical');
|
||||
|
||||
expect(seen[0].fields.status).toBe('500');
|
||||
expect(seen[0].trace).toBe(error.stack);
|
||||
expect(seen[0].shortId).toMatch(/^[a-z]+-[a-z]+-[a-z]+$/);
|
||||
});
|
||||
|
||||
it('keeps one failing handler from starving the others', async () => {
|
||||
const client = makeClient();
|
||||
client.addAlertHandler(
|
||||
async () => {
|
||||
throw new Error('transport down');
|
||||
},
|
||||
{ name: 'broken' },
|
||||
);
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('boom', 'failed');
|
||||
await Promise.resolve();
|
||||
|
||||
expect(seen).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('suppresses alarms once draining', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.onServerPrepareShutdown();
|
||||
client.create('boom', 'too late');
|
||||
|
||||
expect(seen).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient transport registration', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
pdEvent.mockClear();
|
||||
});
|
||||
|
||||
it('splits info to Slack and everything above it to PagerDuty', async () => {
|
||||
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const client = makeClient({
|
||||
pagerduty: { enabled: true, routingKey: 'rk' },
|
||||
slack: { enabled: true, webhookUrl: 'https://hooks.example/abc' },
|
||||
});
|
||||
await client.onServerStart();
|
||||
|
||||
client.create('quiet:thing', 'informational', {}, 'info');
|
||||
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1));
|
||||
expect(pdEvent).not.toHaveBeenCalled();
|
||||
|
||||
client.create('loud:thing', 'paging', {}, 'critical');
|
||||
await vi.waitFor(() => expect(pdEvent).toHaveBeenCalledTimes(1));
|
||||
// The pager has it; chat doesn't repeat it.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('sends every severity to Slack when there is no pager', async () => {
|
||||
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const client = makeClient({
|
||||
slack: { enabled: true, webhookUrl: 'https://hooks.example/abc' },
|
||||
});
|
||||
await client.onServerStart();
|
||||
|
||||
client.create(
|
||||
'loud:thing',
|
||||
'nowhere else to send this',
|
||||
{},
|
||||
'critical',
|
||||
);
|
||||
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1));
|
||||
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('lets config widen the Slack ceiling back out', async () => {
|
||||
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const client = makeClient({
|
||||
pagerduty: { enabled: true, routingKey: 'rk' },
|
||||
slack: {
|
||||
enabled: true,
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
maxSeverity: 'critical',
|
||||
},
|
||||
});
|
||||
await client.onServerStart();
|
||||
|
||||
client.create('loud:thing', 'paging', {}, 'critical');
|
||||
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1));
|
||||
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('gives each occurrence its own PagerDuty incident by default', async () => {
|
||||
const client = makeClient({
|
||||
pagerduty: { enabled: true, routingKey: 'rk' },
|
||||
});
|
||||
await client.onServerStart();
|
||||
|
||||
client.create('scan:failed', 'first', {}, 'warning');
|
||||
client.create('scan:failed', 'second', {}, 'warning');
|
||||
await vi.waitFor(() => expect(pdEvent).toHaveBeenCalledTimes(2));
|
||||
|
||||
const keys = pdEvent.mock.calls.map(
|
||||
([arg]: [{ data: { dedup_key: string } }]) => arg.data.dedup_key,
|
||||
);
|
||||
expect(keys[0]).not.toBe(keys[1]);
|
||||
});
|
||||
|
||||
it('collapses repeats of a dedup alarm onto one incident', async () => {
|
||||
const client = makeClient({
|
||||
pagerduty: { enabled: true, routingKey: 'rk' },
|
||||
});
|
||||
await client.onServerStart();
|
||||
|
||||
const raise = () =>
|
||||
client.create(
|
||||
'http_500:POST:/notif/mark-ack:deadlock',
|
||||
'HTTP 500 on POST /notif/mark-ack: deadlock',
|
||||
{},
|
||||
'critical',
|
||||
{ dedup: true },
|
||||
);
|
||||
raise();
|
||||
raise();
|
||||
await vi.waitFor(() => expect(pdEvent).toHaveBeenCalledTimes(2));
|
||||
|
||||
const keys = pdEvent.mock.calls.map(
|
||||
([arg]: [{ data: { dedup_key: string } }]) => arg.data.dedup_key,
|
||||
);
|
||||
expect(keys[0]).toBe('http_500:POST:/notif/mark-ack:deadlock');
|
||||
expect(keys[1]).toBe(keys[0]);
|
||||
});
|
||||
|
||||
it('skips transports that are enabled but not configured', async () => {
|
||||
const client = makeClient({
|
||||
pagerduty: { enabled: true },
|
||||
slack: { enabled: true },
|
||||
});
|
||||
await client.onServerStart();
|
||||
|
||||
const seen = capture(client);
|
||||
client.create('boom', 'nowhere to send this');
|
||||
|
||||
// Only the capturing handler is registered.
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(pdEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient alarm registry', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
it('finds an alarm by its full id or its short id', () => {
|
||||
const client = makeClient();
|
||||
client.create('disk-pressure', 'nearly full');
|
||||
|
||||
const alarm = client.get('disk-pressure');
|
||||
expect(alarm?.message).toBe('nearly full');
|
||||
expect(client.get(alarm!.shortId)).toBe(alarm);
|
||||
});
|
||||
|
||||
it('returns nothing for an id it has never seen', () => {
|
||||
expect(makeClient().get('never-raised')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('forgets an alarm once cleared, under both ids', () => {
|
||||
const client = makeClient();
|
||||
client.create('disk-pressure', 'nearly full');
|
||||
const shortId = client.get('disk-pressure')!.shortId;
|
||||
|
||||
client.clear('disk-pressure');
|
||||
|
||||
expect(client.get('disk-pressure')).toBeUndefined();
|
||||
expect(client.get(shortId)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps the short id of another alarm that shares it when one is cleared', () => {
|
||||
const client = makeClient();
|
||||
// These two ids hash to the same short id.
|
||||
client.create('alarm:274', 'first');
|
||||
const shortId = client.get('alarm:274')!.shortId;
|
||||
client.create('alarm:643', 'second');
|
||||
expect(client.get('alarm:643')!.shortId).toBe(shortId);
|
||||
|
||||
client.clear('alarm:274');
|
||||
|
||||
expect(client.get(shortId)?.id).toBe('alarm:643');
|
||||
});
|
||||
|
||||
it('ignores a clear for an alarm that is not active', () => {
|
||||
const client = makeClient();
|
||||
expect(() => client.clear('never-raised')).not.toThrow();
|
||||
});
|
||||
|
||||
it('abbreviates long ids in the log line but keeps the short id usable', () => {
|
||||
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
const client = makeClient();
|
||||
const longId = 'a-very-long-alarm-identifier-that-wraps';
|
||||
|
||||
client.create(longId, 'noisy');
|
||||
|
||||
const shortId = client.get(longId)!.shortId;
|
||||
expect(log).toHaveBeenCalledWith(
|
||||
`[alarm] ACTIVE ${shortId} (${longId.slice(0, 20)}...) :: noisy`,
|
||||
);
|
||||
});
|
||||
|
||||
it('accumulates fields across repeats of the same alarm', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('flap', 'first', { a: 1 });
|
||||
client.create('flap', 'second', { b: 2 });
|
||||
|
||||
expect(seen[1].fields).toEqual({ a: '1', b: '2' });
|
||||
expect(client.get('flap')?.occurrences).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('caps retained occurrences while still counting every repeat', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
for (let i = 0; i < 500; i++) {
|
||||
client.create('hot', `occurrence ${i}`, { i });
|
||||
}
|
||||
|
||||
const alarm = client.get('hot')!;
|
||||
expect(alarm.count).toBe(500);
|
||||
expect(alarm.occurrences).toHaveLength(20);
|
||||
expect(alarm.timestamps).toHaveLength(20);
|
||||
// The window kept is the most recent one, not the oldest.
|
||||
expect(alarm.occurrences[19].message).toBe('occurrence 499');
|
||||
// Trimming history must not rewind what the transports are told.
|
||||
expect(seen.at(-1)).toMatchObject({ repeatCount: 500, isRepeat: true });
|
||||
});
|
||||
|
||||
it('names anonymous handlers by their registration order', () => {
|
||||
const client = makeClient();
|
||||
client.addAlertHandler(async () => {
|
||||
throw new Error('down');
|
||||
});
|
||||
expect(() => client.create('boom', 'x')).not.toThrow();
|
||||
});
|
||||
|
||||
it('only logs the drain notice once', () => {
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
log.mockClear();
|
||||
const client = makeClient();
|
||||
|
||||
client.onServerPrepareShutdown();
|
||||
// A second prepare must not restart the drain.
|
||||
client.onServerPrepareShutdown();
|
||||
client.create('a', 'x');
|
||||
client.create('b', 'y');
|
||||
|
||||
const drainLogs = log.mock.calls.filter(
|
||||
([message]) =>
|
||||
message === '[alarm] suppressing alarm while draining',
|
||||
);
|
||||
expect(drainLogs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('substitutes placeholders for an empty id and message', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('', '');
|
||||
|
||||
expect(seen[0]).toMatchObject({
|
||||
id: 'something-bad',
|
||||
message: 'something bad happened',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient alarm eviction', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('forgets alarms that stay quiet past the idle window, under both ids', () => {
|
||||
const client = makeClient();
|
||||
const ids = Array.from(
|
||||
{ length: 50 },
|
||||
(_, i) => `metering_usage_dropped:user-${i}`,
|
||||
);
|
||||
for (const id of ids) client.create(id, 'usage dropped');
|
||||
const shortIds = ids.map((id) => client.get(id)!.shortId);
|
||||
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS + 1);
|
||||
client.create('unrelated', 'the next alarm sweeps the registry');
|
||||
|
||||
expect(ids.filter((id) => client.get(id))).toEqual([]);
|
||||
expect(shortIds.filter((shortId) => client.get(shortId))).toEqual([]);
|
||||
expect(client.get('unrelated')).toBeDefined();
|
||||
});
|
||||
|
||||
it('treats a recurrence after the idle window as a new alarm', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('scan:failed', 'first');
|
||||
client.create('scan:failed', 'second');
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS + 1);
|
||||
client.create('scan:failed', 'back again');
|
||||
|
||||
expect(seen.at(-1)).toMatchObject({ repeatCount: 1, isRepeat: false });
|
||||
// A fresh start keeps per-occurrence incident keys from colliding.
|
||||
expect(seen.at(-1)!.dedupKey).not.toBe(seen[0].dedupKey);
|
||||
});
|
||||
|
||||
it('still treats a repeat inside the window as a repeat after a sweep', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
for (let i = 0; i < 10; i++) client.create('flap', 'same fault');
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS - 1000);
|
||||
client.create('other', 'sweeps the registry');
|
||||
client.create('flap', 'same fault');
|
||||
|
||||
// Occurrence 11 is past the burst, so nothing new goes out for it.
|
||||
expect(seen.map((alert) => alert.id)).toEqual([
|
||||
...Array(10).fill('flap'),
|
||||
'other',
|
||||
]);
|
||||
expect(client.get('flap')?.count).toBe(11);
|
||||
});
|
||||
|
||||
it('keeps an alarm that keeps recurring for longer than the window', () => {
|
||||
const client = makeClient();
|
||||
|
||||
for (let i = 0; i < 6; i++) {
|
||||
client.create('hot', 'still broken');
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS / 2);
|
||||
client.create(`noise:${i}`, 'sweeps the registry');
|
||||
}
|
||||
|
||||
expect(client.get('hot')?.count).toBe(6);
|
||||
});
|
||||
|
||||
it('drops the least recently raised alarm once full', () => {
|
||||
const client = makeClient();
|
||||
|
||||
for (let i = 0; i < MAX_TRACKED_ALARMS; i++) {
|
||||
client.create(`user:${i}`, 'per-user alarm');
|
||||
vi.advanceTimersByTime(1);
|
||||
}
|
||||
// Raised again, so user:1 is now the one seen longest ago.
|
||||
client.create('user:0', 'per-user alarm');
|
||||
client.create('one-more', 'over the cap');
|
||||
|
||||
expect(client.get('user:1')).toBeUndefined();
|
||||
expect(client.get('user:0')?.count).toBe(2);
|
||||
expect(client.get('user:2')).toBeDefined();
|
||||
expect(client.get('one-more')).toBeDefined();
|
||||
});
|
||||
|
||||
it('never holds more than the cap', () => {
|
||||
const client = makeClient();
|
||||
const ids = Array.from(
|
||||
{ length: MAX_TRACKED_ALARMS * 3 },
|
||||
(_, i) => `user:${i}`,
|
||||
);
|
||||
|
||||
for (const id of ids) client.create(id, 'per-user alarm');
|
||||
|
||||
const held = ids.filter((id) => client.get(id));
|
||||
expect(held).toHaveLength(MAX_TRACKED_ALARMS);
|
||||
expect(held[0]).toBe(`user:${MAX_TRACKED_ALARMS * 2}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient known-error rules', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
it('suppresses alerts for a matching rule', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
client.setKnownErrors([
|
||||
{ match: { id: 'noisy' }, action: { type: 'no-alert' } },
|
||||
]);
|
||||
|
||||
client.create('noisy', 'ignore me');
|
||||
client.create('noisy', 'ignore me again');
|
||||
|
||||
expect(seen).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('leaves alarms with a different id alone', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
client.setKnownErrors([
|
||||
{ match: { id: 'noisy' }, action: { type: 'no-alert' } },
|
||||
]);
|
||||
|
||||
client.create('real', 'page me');
|
||||
|
||||
expect(seen).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('only matches when the message matches too', () => {
|
||||
const rules = [
|
||||
{
|
||||
match: { id: 'timeout', message: 'upstream timed out' },
|
||||
action: { type: 'no-alert' as const },
|
||||
},
|
||||
];
|
||||
|
||||
// Separate clients: suppression is recorded on the alarm, so a
|
||||
// second occurrence of the same id would inherit it.
|
||||
const matching = makeClient();
|
||||
const matchingSeen = capture(matching);
|
||||
matching.setKnownErrors(rules);
|
||||
matching.create('timeout', 'upstream timed out');
|
||||
expect(matchingSeen).toHaveLength(0);
|
||||
|
||||
const differing = makeClient();
|
||||
const differingSeen = capture(differing);
|
||||
differing.setKnownErrors(rules);
|
||||
differing.create('timeout', 'something else entirely');
|
||||
expect(differingSeen.map((alert) => alert.message)).toEqual([
|
||||
'something else entirely',
|
||||
]);
|
||||
});
|
||||
|
||||
it('only matches when every named field matches', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
client.setKnownErrors([
|
||||
{
|
||||
match: { id: 'http', fields: { status: 404 } },
|
||||
action: { type: 'no-alert' },
|
||||
},
|
||||
]);
|
||||
|
||||
client.create('http', 'not found', { status: 404 });
|
||||
client.create('http-2', 'server error', { status: 500 });
|
||||
|
||||
expect(seen.map((alert) => alert.id)).toEqual(['http-2']);
|
||||
});
|
||||
|
||||
it('retiers a matching alarm to a lower severity', () => {
|
||||
const client = makeClient();
|
||||
const paging = capture(client, 'error');
|
||||
const chat = capture(client, 'info');
|
||||
client.setKnownErrors([
|
||||
{
|
||||
match: { id: 'flaky' },
|
||||
action: { type: 'severity', value: 'info' },
|
||||
},
|
||||
]);
|
||||
|
||||
client.create('flaky', 'transient', {}, 'critical');
|
||||
|
||||
expect(paging).toHaveLength(0);
|
||||
expect(chat).toHaveLength(1);
|
||||
expect(chat[0].severity).toBe('info');
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient repeat reporting', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
pdEvent.mockClear();
|
||||
});
|
||||
|
||||
it('reports every occurrence up to the burst, then backs off', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
for (let i = 0; i < 12; i++) client.create('flap', 'same fault');
|
||||
|
||||
expect(seen).toHaveLength(10);
|
||||
expect(seen[9].repeatCount).toBe(10);
|
||||
});
|
||||
|
||||
it('reports again once the interval is reached', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
for (let i = 0; i < 500; i++) client.create('flap', 'same fault');
|
||||
|
||||
expect(seen).toHaveLength(11);
|
||||
expect(seen[10].repeatCount).toBe(500);
|
||||
});
|
||||
|
||||
it('keeps a fast-repeating fault out of the log after the burst', () => {
|
||||
const client = makeClient();
|
||||
capture(client);
|
||||
const warn = vi.mocked(console.warn);
|
||||
warn.mockClear();
|
||||
|
||||
for (let i = 0; i < 100; i++) client.create('flap', 'same fault');
|
||||
|
||||
// Occurrences 2-10 only; the first is an ACTIVE line on console.error.
|
||||
expect(warn.mock.calls).toHaveLength(9);
|
||||
});
|
||||
|
||||
it('skips the payload entirely when no transport accepts the severity', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client, 'critical');
|
||||
|
||||
client.create('quiet', 'nobody is listening', {}, 'info');
|
||||
|
||||
expect(seen).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
/*
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
@@ -20,84 +20,64 @@
|
||||
import { event as pdEvent } from '@pagerduty/pdjs';
|
||||
import { inspect } from 'node:util';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { IConfig, PagerSeverity, SeverityRule } from '../../types';
|
||||
import type { IConfig } from '../../types';
|
||||
import { PuterClient } from '../types';
|
||||
import {
|
||||
meetsMinSeverity,
|
||||
resolveSeverityOverride,
|
||||
withinMaxSeverity,
|
||||
} from './severity';
|
||||
import { createSlackAlertHandler } from './slack';
|
||||
import type {
|
||||
Alarm,
|
||||
AlarmFields,
|
||||
AlarmOptions,
|
||||
AlertHandler,
|
||||
AlertPayload,
|
||||
KnownErrorRule,
|
||||
} from './types';
|
||||
|
||||
export type {
|
||||
Alarm,
|
||||
AlarmFields,
|
||||
AlarmOptions,
|
||||
AlertHandler,
|
||||
AlertPayload,
|
||||
KnownErrorRule,
|
||||
} from './types';
|
||||
export type { PagerSeverity } from '../../types';
|
||||
// ── Types ────────────────────────────────────────────────────────────
|
||||
|
||||
// -- Types ------------------------------------------------------------
|
||||
|
||||
interface RegisteredHandler {
|
||||
name: string;
|
||||
/** Lowest severity this transport accepts. */
|
||||
minSeverity: PagerSeverity;
|
||||
/** Highest severity this transport accepts. */
|
||||
maxSeverity: PagerSeverity;
|
||||
handler: AlertHandler;
|
||||
export interface AlarmFields {
|
||||
error?: Error;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** Severity used when neither the call site nor config picks one. */
|
||||
const FALLBACK_SEVERITY: PagerSeverity = 'critical';
|
||||
/**
|
||||
* How many recent occurrences an alarm keeps. A hot alarm repeats for as long
|
||||
* as the fault lasts, so retaining every occurrence means retaining every
|
||||
* message and field set it was ever raised with, request bodies and actors
|
||||
* included. The last few are what a human reads; the rest is only a count, and
|
||||
* `count` keeps that.
|
||||
*/
|
||||
const OCCURRENCE_HISTORY_LIMIT = 20;
|
||||
/**
|
||||
* An alarm that hasn't recurred for this long is forgotten; its next occurrence
|
||||
* is a new alarm with a fresh count. Kept far past the chat transport's repeat
|
||||
* throttle so a slowly recurring fault still counts as one alarm.
|
||||
*/
|
||||
export const ALARM_IDLE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
/**
|
||||
* Most alarms held at once. Ids that embed a user or key can outrun the idle
|
||||
* window; past this the least recently raised alarm is dropped.
|
||||
*/
|
||||
export const MAX_TRACKED_ALARMS = 1000;
|
||||
interface AlarmOccurrence {
|
||||
message: string;
|
||||
fields: AlarmFields;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
// A fast-recurring fault repeats far faster than it is useful to report, and
|
||||
// every report allocates a log line and a payload. Report every occurrence up
|
||||
// to the burst, then one per interval; `count` still tracks all of them.
|
||||
const REPEAT_REPORT_BURST = 10;
|
||||
const REPEAT_REPORT_INTERVAL = 500;
|
||||
interface Alarm {
|
||||
id: string;
|
||||
shortId: string;
|
||||
message: string;
|
||||
fields: AlarmFields;
|
||||
error?: Error;
|
||||
started: number;
|
||||
timestamps: number[];
|
||||
occurrences: AlarmOccurrence[];
|
||||
severity?: PagerSeverity;
|
||||
noAlert?: boolean;
|
||||
}
|
||||
|
||||
const shouldReportRepeat = (count: number): boolean =>
|
||||
count <= REPEAT_REPORT_BURST || count % REPEAT_REPORT_INTERVAL === 0;
|
||||
/** Keeps `info` alarms out of the paging system unless config says otherwise. */
|
||||
const DEFAULT_PAGERDUTY_MIN_SEVERITY: PagerSeverity = 'warning';
|
||||
/** Slack's ceiling once a pager exists: chat gets what doesn't page. */
|
||||
const DEFAULT_SLACK_MAX_SEVERITY_WITH_PAGER: PagerSeverity = 'info';
|
||||
type PagerSeverity = 'critical' | 'error' | 'warning' | 'info';
|
||||
|
||||
// -- Helpers ----------------------------------------------------------
|
||||
export interface AlertPayload {
|
||||
id: string;
|
||||
message: string;
|
||||
source: string;
|
||||
severity: PagerSeverity;
|
||||
custom?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
type AlertHandler = (alert: AlertPayload) => Promise<void>;
|
||||
|
||||
interface KnownErrorRule {
|
||||
match: {
|
||||
id: string;
|
||||
message?: string;
|
||||
fields?: Record<string, unknown>;
|
||||
};
|
||||
action: {
|
||||
type: 'no-alert' | 'severity';
|
||||
value?: PagerSeverity;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Deterministic short identifier derived from an alarm ID. Produces a readable
|
||||
* 3-word slug like "amber-delta-fox".
|
||||
* Deterministic short identifier derived from an alarm ID.
|
||||
* Produces a readable 3-word slug like "amber-delta-fox".
|
||||
*/
|
||||
const WORD_POOL = [
|
||||
'alpha',
|
||||
@@ -186,21 +166,16 @@ function cleanFields(fields: AlarmFields): Record<string, string> {
|
||||
return out;
|
||||
}
|
||||
|
||||
// -- AlarmClient ------------------------------------------------------
|
||||
// ── AlarmClient ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Manages system alarms and routes them to alert transports by severity.
|
||||
*
|
||||
* Severity is the routing decision: each transport declares the severity window
|
||||
* it accepts, so `critical` pages on-call while `info` only reaches the chat
|
||||
* channel. Config can retier or mute any alarm id after the fact — see
|
||||
* `pager.severityOverrides` in {@link IConfig}.
|
||||
* Manages system alarms and dispatches alerts to external paging
|
||||
* services (PagerDuty, or any registered handler).
|
||||
*/
|
||||
export class AlarmClient extends PuterClient {
|
||||
/** Ordered by last occurrence: a repeat moves its alarm to the back. */
|
||||
private alarms = new Map<string, Alarm>();
|
||||
private aliases = new Map<string, Alarm>();
|
||||
private alertHandlers: RegisteredHandler[] = [];
|
||||
private alertHandlers: AlertHandler[] = [];
|
||||
private knownErrors: KnownErrorRule[] = [];
|
||||
private draining = false;
|
||||
private drainLogged = false;
|
||||
@@ -209,11 +184,42 @@ export class AlarmClient extends PuterClient {
|
||||
super(config);
|
||||
}
|
||||
|
||||
// -- Lifecycle ----------------------------------------------------
|
||||
// ── Lifecycle ────────────────────────────────────────────────────
|
||||
|
||||
override async onServerStart(): Promise<void> {
|
||||
const paging = this.registerPagerDuty();
|
||||
this.registerSlack({ paging });
|
||||
const pagerConf = this.config.pager;
|
||||
if (!pagerConf?.pagerduty?.enabled) return;
|
||||
|
||||
const routingKey = pagerConf.pagerduty.routingKey;
|
||||
if (!routingKey) {
|
||||
console.warn(
|
||||
'[alarm] PagerDuty enabled but no routingKey configured',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const serverId = this.config.serverId;
|
||||
|
||||
this.alertHandlers.push(async (alert) => {
|
||||
await pdEvent({
|
||||
data: {
|
||||
routing_key: routingKey,
|
||||
event_action: 'trigger',
|
||||
dedup_key: alert.id,
|
||||
payload: {
|
||||
summary: alert.message,
|
||||
source: alert.source,
|
||||
severity: alert.severity,
|
||||
custom_details: {
|
||||
...alert.custom,
|
||||
server_id: serverId,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
console.log('[alarm] PagerDuty handler registered');
|
||||
}
|
||||
|
||||
override onServerPrepareShutdown(): void {
|
||||
@@ -222,109 +228,13 @@ export class AlarmClient extends PuterClient {
|
||||
console.log('[alarm] entering drain mode — suppressing new alarms');
|
||||
}
|
||||
|
||||
/** @returns Whether a working PagerDuty transport was registered. */
|
||||
private registerPagerDuty(): boolean {
|
||||
const pagerDutyConf = this.config.pager?.pagerduty;
|
||||
if (!pagerDutyConf?.enabled) return false;
|
||||
|
||||
const routingKey = pagerDutyConf.routingKey;
|
||||
if (!routingKey) {
|
||||
console.warn(
|
||||
'[alarm] PagerDuty enabled but no routingKey configured',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const serverId = this.config.serverId;
|
||||
const minSeverity =
|
||||
pagerDutyConf.minSeverity ?? DEFAULT_PAGERDUTY_MIN_SEVERITY;
|
||||
|
||||
this.addAlertHandler(
|
||||
async (alert) => {
|
||||
await pdEvent({
|
||||
data: {
|
||||
routing_key: routingKey,
|
||||
event_action: 'trigger',
|
||||
dedup_key: alert.dedupKey,
|
||||
payload: {
|
||||
summary: alert.message,
|
||||
source: alert.source,
|
||||
severity: alert.severity,
|
||||
custom_details: {
|
||||
...alert.custom,
|
||||
server_id: serverId,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
{ name: 'pagerduty', minSeverity },
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[alarm] PagerDuty handler registered (min severity: ${minSeverity})`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
private registerSlack({ paging }: { paging: boolean }): void {
|
||||
const slackConf = this.config.pager?.slack;
|
||||
if (!slackConf?.enabled) return;
|
||||
|
||||
if (!slackConf.webhookUrl) {
|
||||
console.warn('[alarm] Slack enabled but no webhookUrl configured');
|
||||
return;
|
||||
}
|
||||
|
||||
const minSeverity = slackConf.minSeverity ?? 'info';
|
||||
// With a pager taking everything from `warning` up, chat is where the
|
||||
// rest is recorded — reposting the paging tiers there only trains
|
||||
// people to skim the channel. Without one, Slack is the only place an
|
||||
// alarm can land, so it takes all of them.
|
||||
const maxSeverity =
|
||||
slackConf.maxSeverity ??
|
||||
(paging ? DEFAULT_SLACK_MAX_SEVERITY_WITH_PAGER : 'critical');
|
||||
|
||||
this.addAlertHandler(
|
||||
createSlackAlertHandler(slackConf, {
|
||||
serverId: this.config.serverId,
|
||||
}),
|
||||
{ name: 'slack', minSeverity, maxSeverity },
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[alarm] Slack handler registered (severity ${minSeverity}..${maxSeverity})`,
|
||||
);
|
||||
}
|
||||
|
||||
// -- Public API ---------------------------------------------------
|
||||
// ── Public API ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create or update an alarm. If the alarm ID already exists, the occurrence
|
||||
* count is incremented and a repeat alert is dispatched. An alarm idle for
|
||||
* {@link ALARM_IDLE_TTL_MS} is forgotten, so its next occurrence is new.
|
||||
*
|
||||
* `severity` decides where the alarm lands:
|
||||
*
|
||||
* Critical — a real outage; pages on-call. Reserve it for unhandled server
|
||||
* errors. error — pages on-call as well; prefer `critical` or `warning`.
|
||||
* warning — worth a look soon, but nobody gets woken up. info — a record in
|
||||
* the chat channel; never pages.
|
||||
*
|
||||
* Omit it to take `pager.defaultSeverity` (itself defaulting to
|
||||
* 'critical'). Operators can retier or mute any alarm id from config
|
||||
* afterwards, so the value here is the starting point, not the last word.
|
||||
*
|
||||
* Pass `{ dedup: true }` when repeats of this id are one recurring fault
|
||||
* that should collapse into a single incident — see {@link AlarmOptions}.
|
||||
* Create or update an alarm. If the alarm ID already exists, the
|
||||
* occurrence count is incremented and a repeat alert is dispatched.
|
||||
*/
|
||||
create(
|
||||
id: string,
|
||||
message: string,
|
||||
fields: AlarmFields = {},
|
||||
severity?: PagerSeverity,
|
||||
opts: AlarmOptions = {},
|
||||
): void {
|
||||
create(id: string, message: string, fields: AlarmFields = {}): void {
|
||||
if (this.draining) {
|
||||
if (!this.drainLogged) {
|
||||
this.drainLogged = true;
|
||||
@@ -333,12 +243,9 @@ export class AlarmClient extends PuterClient {
|
||||
return;
|
||||
}
|
||||
|
||||
this.evict();
|
||||
const existing = this.alarms.get(id);
|
||||
|
||||
if (existing) {
|
||||
this.alarms.delete(id);
|
||||
this.alarms.set(id, existing);
|
||||
this.recordOccurrence(existing, message, fields);
|
||||
this.handleRepeat(existing);
|
||||
return;
|
||||
@@ -349,13 +256,8 @@ export class AlarmClient extends PuterClient {
|
||||
shortId: shortId(id),
|
||||
message,
|
||||
fields,
|
||||
severity,
|
||||
dedup: opts.dedup,
|
||||
started: Date.now(),
|
||||
// `recordOccurrence` below stamps the first occurrence; seeding one
|
||||
// here too would report every alarm as one occurrence ahead.
|
||||
count: 0,
|
||||
timestamps: [],
|
||||
timestamps: [Date.now()],
|
||||
occurrences: [],
|
||||
};
|
||||
if (fields.error) alarm.error = fields.error;
|
||||
@@ -363,7 +265,6 @@ export class AlarmClient extends PuterClient {
|
||||
this.alarms.set(id, alarm);
|
||||
this.aliases.set(alarm.shortId, alarm);
|
||||
this.recordOccurrence(alarm, message, fields);
|
||||
this.evict();
|
||||
this.handleNew(alarm);
|
||||
}
|
||||
|
||||
@@ -372,7 +273,8 @@ export class AlarmClient extends PuterClient {
|
||||
const alarm = this.alarms.get(id);
|
||||
if (!alarm) return;
|
||||
|
||||
this.forget(alarm);
|
||||
this.alarms.delete(id);
|
||||
this.aliases.delete(alarm.shortId);
|
||||
console.log(`[alarm] CLEAR ${displayId(alarm)} :: ${alarm.message}`);
|
||||
}
|
||||
|
||||
@@ -382,84 +284,37 @@ export class AlarmClient extends PuterClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Register an additional alert handler. Handlers are called for every alarm
|
||||
* that isn't suppressed by a known-error rule or muted by config, and whose
|
||||
* severity falls inside the handler's own `minSeverity`..`maxSeverity`
|
||||
* window (default: everything).
|
||||
* Register an additional alert handler. Handlers are called for
|
||||
* every alarm that isn't suppressed by a known-error rule.
|
||||
*/
|
||||
addAlertHandler(
|
||||
handler: AlertHandler,
|
||||
opts: {
|
||||
name?: string;
|
||||
minSeverity?: PagerSeverity;
|
||||
maxSeverity?: PagerSeverity;
|
||||
} = {},
|
||||
): void {
|
||||
this.alertHandlers.push({
|
||||
name: opts.name ?? `handler-${this.alertHandlers.length}`,
|
||||
minSeverity: opts.minSeverity ?? 'info',
|
||||
maxSeverity: opts.maxSeverity ?? 'critical',
|
||||
handler,
|
||||
});
|
||||
addAlertHandler(handler: AlertHandler): void {
|
||||
this.alertHandlers.push(handler);
|
||||
}
|
||||
|
||||
/** Add rules that can suppress or adjust severity of known errors. */
|
||||
/**
|
||||
* Add rules that can suppress or adjust severity of known errors.
|
||||
*/
|
||||
setKnownErrors(rules: KnownErrorRule[]): void {
|
||||
this.knownErrors = rules;
|
||||
}
|
||||
|
||||
// -- Internals ----------------------------------------------------
|
||||
|
||||
/**
|
||||
* Drop idle alarms, then the least recently raised ones past the cap. The
|
||||
* map's order puts both at the front, so this stops at the first keeper.
|
||||
*/
|
||||
private evict(): void {
|
||||
const now = Date.now();
|
||||
for (const alarm of this.alarms.values()) {
|
||||
const lastSeen = alarm.timestamps.at(-1) ?? alarm.started;
|
||||
if (
|
||||
this.alarms.size <= MAX_TRACKED_ALARMS &&
|
||||
now - lastSeen < ALARM_IDLE_TTL_MS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
this.forget(alarm);
|
||||
}
|
||||
}
|
||||
|
||||
private forget(alarm: Alarm): void {
|
||||
this.alarms.delete(alarm.id);
|
||||
// Short ids can collide; leave an alias that now names another alarm.
|
||||
if (this.aliases.get(alarm.shortId) === alarm) {
|
||||
this.aliases.delete(alarm.shortId);
|
||||
}
|
||||
}
|
||||
// ── Internals ────────────────────────────────────────────────────
|
||||
|
||||
private recordOccurrence(
|
||||
alarm: Alarm,
|
||||
message: string,
|
||||
fields: AlarmFields,
|
||||
): void {
|
||||
const now = Date.now();
|
||||
alarm.message = message;
|
||||
alarm.fields = { ...alarm.fields, ...fields };
|
||||
alarm.count++;
|
||||
alarm.timestamps.push(Date.now());
|
||||
if (fields.error) alarm.error = fields.error;
|
||||
|
||||
alarm.timestamps.push(now);
|
||||
alarm.occurrences.push({ message, fields, timestamp: now });
|
||||
|
||||
if (alarm.timestamps.length > OCCURRENCE_HISTORY_LIMIT) {
|
||||
alarm.timestamps.splice(
|
||||
0,
|
||||
alarm.timestamps.length - OCCURRENCE_HISTORY_LIMIT,
|
||||
);
|
||||
alarm.occurrences.splice(
|
||||
0,
|
||||
alarm.occurrences.length - OCCURRENCE_HISTORY_LIMIT,
|
||||
);
|
||||
}
|
||||
alarm.occurrences.push({
|
||||
message,
|
||||
fields,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
private applyKnownErrors(alarm: Alarm): void {
|
||||
@@ -506,10 +361,8 @@ export class AlarmClient extends PuterClient {
|
||||
private handleRepeat(alarm: Alarm): void {
|
||||
this.applyKnownErrors(alarm);
|
||||
|
||||
if (!shouldReportRepeat(alarm.count)) return;
|
||||
|
||||
console.warn(
|
||||
`[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.count})`,
|
||||
`[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.timestamps.length})`,
|
||||
);
|
||||
|
||||
if (alarm.noAlert) return;
|
||||
@@ -517,88 +370,25 @@ export class AlarmClient extends PuterClient {
|
||||
this.dispatchAlert(alarm);
|
||||
}
|
||||
|
||||
/**
|
||||
* Call-site severity, then any known-error rule (both already on the
|
||||
* alarm), then the config override — so an operator always has the last
|
||||
* word over what the code asked for.
|
||||
*/
|
||||
private resolveSeverity(alarm: Alarm): SeverityRule {
|
||||
const base =
|
||||
alarm.severity ??
|
||||
this.config.pager?.defaultSeverity ??
|
||||
FALLBACK_SEVERITY;
|
||||
return (
|
||||
resolveSeverityOverride(
|
||||
alarm.id,
|
||||
this.config.pager?.severityOverrides,
|
||||
) ?? base
|
||||
);
|
||||
}
|
||||
|
||||
private hasHandlerFor(severity: PagerSeverity): boolean {
|
||||
for (const { minSeverity, maxSeverity } of this.alertHandlers) {
|
||||
if (
|
||||
meetsMinSeverity(severity, minSeverity) &&
|
||||
withinMaxSeverity(severity, maxSeverity)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private dispatchAlert(alarm: Alarm): void {
|
||||
const resolved = this.resolveSeverity(alarm);
|
||||
if (resolved === 'mute') {
|
||||
if (!alarm.muteLogged) {
|
||||
alarm.muteLogged = true;
|
||||
console.log(`[alarm] MUTED by config ${displayId(alarm)}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
alarm.severity = resolved;
|
||||
|
||||
// Build nothing for a severity no transport accepts — the payload
|
||||
// (cleaned fields plus the stack, twice) is the expensive part.
|
||||
if (!this.hasHandlerFor(resolved)) return;
|
||||
|
||||
const severity = alarm.severity ?? 'critical';
|
||||
const fieldsClean = cleanFields(alarm.fields);
|
||||
const repeatCount = alarm.count;
|
||||
|
||||
const id = alarm.id || 'something-bad';
|
||||
|
||||
const payload: AlertPayload = {
|
||||
id,
|
||||
// A de-duplicating transport should fold repeats of a recurring
|
||||
// fault into one incident, but everything else is a fresh event
|
||||
// each time it fires — the start time keeps occurrence numbering
|
||||
// from colliding with an incident left open by an earlier boot.
|
||||
dedupKey: alarm.dedup
|
||||
? id
|
||||
: `${id}#${alarm.started}.${repeatCount}`,
|
||||
shortId: alarm.shortId,
|
||||
id: alarm.id || 'something-bad',
|
||||
message: alarm.message || alarm.id || 'something bad happened',
|
||||
source: 'alarm',
|
||||
severity: resolved,
|
||||
fields: fieldsClean,
|
||||
trace: alarm.error?.stack,
|
||||
repeatCount,
|
||||
isRepeat: repeatCount > 1,
|
||||
severity,
|
||||
custom: {
|
||||
fields: fieldsClean,
|
||||
trace: alarm.error?.stack,
|
||||
repeat_count: repeatCount,
|
||||
repeat_count: alarm.timestamps.length,
|
||||
},
|
||||
};
|
||||
|
||||
for (const { name, minSeverity, maxSeverity, handler } of this
|
||||
.alertHandlers) {
|
||||
if (!meetsMinSeverity(resolved, minSeverity)) continue;
|
||||
if (!withinMaxSeverity(resolved, maxSeverity)) continue;
|
||||
for (const handler of this.alertHandlers) {
|
||||
handler(payload).catch((err) => {
|
||||
console.error(
|
||||
`[alarm] ${name} alert handler failed: ${err?.message}`,
|
||||
);
|
||||
console.error(`[alarm] alert handler failed: ${err?.message}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { meetsMinSeverity, resolveSeverityOverride } from './severity';
|
||||
|
||||
describe('meetsMinSeverity', () => {
|
||||
it('accepts everything at or above the floor', () => {
|
||||
expect(meetsMinSeverity('critical', 'warning')).toBe(true);
|
||||
expect(meetsMinSeverity('error', 'warning')).toBe(true);
|
||||
expect(meetsMinSeverity('warning', 'warning')).toBe(true);
|
||||
expect(meetsMinSeverity('info', 'warning')).toBe(false);
|
||||
});
|
||||
|
||||
it('lets an info floor take every severity', () => {
|
||||
for (const severity of [
|
||||
'critical',
|
||||
'error',
|
||||
'warning',
|
||||
'info',
|
||||
] as const) {
|
||||
expect(meetsMinSeverity(severity, 'info')).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveSeverityOverride', () => {
|
||||
it('returns undefined without overrides or on no match', () => {
|
||||
expect(resolveSeverityOverride('a:b', undefined)).toBeUndefined();
|
||||
expect(
|
||||
resolveSeverityOverride('a:b', { 'c:*': 'info' }),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('matches an exact id', () => {
|
||||
expect(
|
||||
resolveSeverityOverride('cronMonitor:lowSignupRate', {
|
||||
'cronMonitor:lowSignupRate': 'info',
|
||||
}),
|
||||
).toBe('info');
|
||||
});
|
||||
|
||||
it('matches a prefix pattern', () => {
|
||||
expect(
|
||||
resolveSeverityOverride('cronMonitor:high_aiLogEntries', {
|
||||
'cronMonitor:*': 'warning',
|
||||
}),
|
||||
).toBe('warning');
|
||||
});
|
||||
|
||||
it('prefers the exact id over a prefix pattern', () => {
|
||||
expect(
|
||||
resolveSeverityOverride('cronMonitor:lowSignupRate', {
|
||||
'cronMonitor:*': 'warning',
|
||||
'cronMonitor:lowSignupRate': 'mute',
|
||||
}),
|
||||
).toBe('mute');
|
||||
});
|
||||
|
||||
it('prefers the longest matching prefix', () => {
|
||||
expect(
|
||||
resolveSeverityOverride('abuse:card-verification:setup-failed', {
|
||||
'abuse:*': 'info',
|
||||
'abuse:card-verification:*': 'mute',
|
||||
}),
|
||||
).toBe('mute');
|
||||
});
|
||||
|
||||
it('drops an unrecognized rule rather than guessing', () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
expect(
|
||||
resolveSeverityOverride('a:b', {
|
||||
'a:b': 'silent' as unknown as 'mute',
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(warn).toHaveBeenCalled();
|
||||
warn.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -1,94 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { PagerSeverity, SeverityRule } from '../../types';
|
||||
|
||||
/** Ascending urgency. A transport takes everything at or above its floor. */
|
||||
const RANK: Record<PagerSeverity, number> = {
|
||||
info: 0,
|
||||
warning: 1,
|
||||
error: 2,
|
||||
critical: 3,
|
||||
};
|
||||
|
||||
const SEVERITIES = Object.keys(RANK) as PagerSeverity[];
|
||||
|
||||
export function isPagerSeverity(value: unknown): value is PagerSeverity {
|
||||
return typeof value === 'string' && value in RANK;
|
||||
}
|
||||
|
||||
/** True when `severity` is urgent enough for a transport whose floor is `min`. */
|
||||
export function meetsMinSeverity(
|
||||
severity: PagerSeverity,
|
||||
min: PagerSeverity,
|
||||
): boolean {
|
||||
return RANK[severity] >= RANK[min];
|
||||
}
|
||||
|
||||
/**
|
||||
* True when `severity` is quiet enough for a transport whose ceiling is `max`.
|
||||
* A ceiling is what keeps a chat transport from repeating everything the pager
|
||||
* already delivered.
|
||||
*/
|
||||
export function withinMaxSeverity(
|
||||
severity: PagerSeverity,
|
||||
max: PagerSeverity,
|
||||
): boolean {
|
||||
return RANK[severity] <= RANK[max];
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up the operator override for an alarm id. Exact ids win over prefix
|
||||
* patterns (`cronMonitor:*`); among patterns the longest prefix wins, so a
|
||||
* specific rule can carve an exception out of a broad one.
|
||||
*/
|
||||
export function resolveSeverityOverride(
|
||||
id: string,
|
||||
overrides: Record<string, SeverityRule> | undefined,
|
||||
): SeverityRule | undefined {
|
||||
if (!overrides) return undefined;
|
||||
|
||||
const exact = overrides[id];
|
||||
if (exact !== undefined) return validRule(id, exact);
|
||||
|
||||
let bestLength = -1;
|
||||
let best: SeverityRule | undefined;
|
||||
for (const [pattern, rule] of Object.entries(overrides)) {
|
||||
if (!pattern.endsWith('*')) continue;
|
||||
const prefix = pattern.slice(0, -1);
|
||||
if (!id.startsWith(prefix)) continue;
|
||||
if (prefix.length <= bestLength) continue;
|
||||
bestLength = prefix.length;
|
||||
best = rule;
|
||||
}
|
||||
return best === undefined ? undefined : validRule(id, best);
|
||||
}
|
||||
|
||||
/**
|
||||
* A typo in the override map would otherwise silently mute or escalate an
|
||||
* alarm, so an unrecognized value is dropped with a warning instead.
|
||||
*/
|
||||
function validRule(id: string, rule: SeverityRule): SeverityRule | undefined {
|
||||
if (rule === 'mute' || isPagerSeverity(rule)) return rule;
|
||||
console.warn(
|
||||
`[alarm] ignoring invalid severity override "${rule}" for ${id} ` +
|
||||
`(expected mute or one of ${SEVERITIES.join(', ')})`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { buildSlackMessage, createSlackAlertHandler } from './slack';
|
||||
import type { AlertPayload } from './types';
|
||||
|
||||
const alert = (over: Partial<AlertPayload> = {}): AlertPayload => ({
|
||||
id: 'cronMonitor:high_aiLogEntries',
|
||||
dedupKey: 'cronMonitor:high_aiLogEntries#1.1',
|
||||
shortId: 'amber-delta-fox',
|
||||
message: 'High AI log entries: 1200 in the last 10 minutes',
|
||||
source: 'alarm',
|
||||
severity: 'warning',
|
||||
fields: { count: '1200', threshold: '1000' },
|
||||
repeatCount: 1,
|
||||
isRepeat: false,
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('buildSlackMessage', () => {
|
||||
it('renders severity, message and fields', () => {
|
||||
const msg = buildSlackMessage(alert(), {
|
||||
channel: '#alerts',
|
||||
username: 'puter-alarms',
|
||||
serverId: 'oregon',
|
||||
});
|
||||
|
||||
expect(msg.text).toContain('[WARNING]');
|
||||
expect(msg.text).toContain('High AI log entries');
|
||||
expect(msg.channel).toBe('#alerts');
|
||||
expect(msg.username).toBe('puter-alarms');
|
||||
expect(msg.attachments[0].fields).toEqual([
|
||||
{ title: 'count', value: '1200', short: true },
|
||||
{ title: 'threshold', value: '1000', short: true },
|
||||
]);
|
||||
expect(msg.attachments[0].footer).toBe(
|
||||
'amber-delta-fox • cronMonitor:high_aiLogEntries • oregon',
|
||||
);
|
||||
});
|
||||
|
||||
it('marks repeats with an occurrence count', () => {
|
||||
const msg = buildSlackMessage(
|
||||
alert({ repeatCount: 7, isRepeat: true }),
|
||||
);
|
||||
expect(msg.text).toContain('(x7)');
|
||||
});
|
||||
|
||||
it('colours each severity differently', () => {
|
||||
const colors = (['critical', 'error', 'warning', 'info'] as const).map(
|
||||
(severity) =>
|
||||
buildSlackMessage(alert({ severity })).attachments[0].color,
|
||||
);
|
||||
expect(new Set(colors).size).toBe(4);
|
||||
});
|
||||
|
||||
it('puts the stack in a code block and keeps it out of the fields', () => {
|
||||
const msg = buildSlackMessage(
|
||||
alert({
|
||||
fields: { error: 'Error: boom', path: '/api/x' },
|
||||
trace: 'Error: boom\n at handler',
|
||||
}),
|
||||
);
|
||||
expect(msg.attachments[0].text).toBe(
|
||||
'```Error: boom\n at handler```',
|
||||
);
|
||||
expect(msg.attachments[0].fields.map((f) => f.title)).toEqual(['path']);
|
||||
});
|
||||
|
||||
it('truncates long values and caps the field count', () => {
|
||||
const fields: Record<string, string> = { long: 'x'.repeat(1000) };
|
||||
for (let i = 0; i < 30; i++) fields[`f${i}`] = String(i);
|
||||
|
||||
const msg = buildSlackMessage(alert({ fields }));
|
||||
expect(msg.attachments[0].fields.length).toBe(12);
|
||||
expect(msg.attachments[0].fields[0].value).toHaveLength(400);
|
||||
expect(msg.attachments[0].fields[0].value.endsWith('…')).toBe(true);
|
||||
});
|
||||
|
||||
it('omits channel and username when unset', () => {
|
||||
const msg = buildSlackMessage(alert());
|
||||
expect(msg.channel).toBeUndefined();
|
||||
expect(msg.username).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('createSlackAlertHandler', () => {
|
||||
let fetchMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
fetchMock = vi.fn(async () => ({ ok: true, status: 200 }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('posts the payload to the webhook', async () => {
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
channel: '#alerts',
|
||||
});
|
||||
await handler(alert());
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const [url, init] = fetchMock.mock.calls[0];
|
||||
expect(url).toBe('https://hooks.example/abc');
|
||||
expect(init.method).toBe('POST');
|
||||
expect(JSON.parse(init.body).channel).toBe('#alerts');
|
||||
});
|
||||
|
||||
it('throttles repeats of the same alarm and lets other ids through', async () => {
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
repeatThrottleMs: 60_000,
|
||||
});
|
||||
|
||||
await handler(alert());
|
||||
await handler(alert({ repeatCount: 2, isRepeat: true }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
await handler(alert({ id: 'other:alarm' }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
|
||||
vi.advanceTimersByTime(60_000);
|
||||
await handler(alert({ repeatCount: 3, isRepeat: true }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('posts every occurrence when throttling is disabled', async () => {
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
repeatThrottleMs: 0,
|
||||
});
|
||||
|
||||
await handler(alert());
|
||||
await handler(alert({ repeatCount: 2, isRepeat: true }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not let a failed post consume the throttle slot', async () => {
|
||||
fetchMock.mockResolvedValueOnce({ ok: false, status: 500 });
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
repeatThrottleMs: 60_000,
|
||||
});
|
||||
|
||||
await expect(handler(alert())).rejects.toThrow('500');
|
||||
await handler(alert());
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('releases the throttle slot when the post itself errors', async () => {
|
||||
fetchMock.mockRejectedValueOnce(new Error('socket hang up'));
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
repeatThrottleMs: 60_000,
|
||||
});
|
||||
|
||||
await expect(handler(alert())).rejects.toThrow('socket hang up');
|
||||
await handler(alert());
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('evicts throttle entries that have aged out once the map fills', async () => {
|
||||
const throttleMs = 60_000;
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
repeatThrottleMs: throttleMs,
|
||||
});
|
||||
|
||||
// Fill to the high-water mark, then age every entry past the window.
|
||||
for (let i = 0; i < 5000; i++) {
|
||||
await handler(alert({ id: `aged:${i}` }));
|
||||
}
|
||||
vi.advanceTimersByTime(throttleMs);
|
||||
|
||||
await handler(alert({ id: 'fresh' }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5001);
|
||||
|
||||
// The aged ids were pruned, so they post again immediately.
|
||||
await handler(alert({ id: 'aged:0' }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5002);
|
||||
});
|
||||
|
||||
it('drops the oldest entry when the map is full of live ones', async () => {
|
||||
const handler = createSlackAlertHandler({
|
||||
webhookUrl: 'https://hooks.example/abc',
|
||||
repeatThrottleMs: 60_000,
|
||||
});
|
||||
|
||||
for (let i = 0; i < 5000; i++) {
|
||||
await handler(alert({ id: `live:${i}` }));
|
||||
}
|
||||
|
||||
// Nothing has aged out, so the oldest id makes room for the new one.
|
||||
await handler(alert({ id: 'newcomer' }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5001);
|
||||
|
||||
await handler(alert({ id: 'live:0' }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5002);
|
||||
// The newcomer is still throttled.
|
||||
await handler(alert({ id: 'newcomer' }));
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5002);
|
||||
});
|
||||
});
|
||||
@@ -1,163 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { ISlackAlertConfig, PagerSeverity } from '../../types';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap';
|
||||
import type { AlertHandler, AlertPayload } from './types';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 5000;
|
||||
export const DEFAULT_REPEAT_THROTTLE_MS = 15 * 60 * 1000;
|
||||
|
||||
/** Beyond this many tracked alarm ids, throttle state is pruned. */
|
||||
const MAX_THROTTLE_ENTRIES = 5000;
|
||||
|
||||
const MAX_FIELDS = 12;
|
||||
const MAX_FIELD_VALUE = 400;
|
||||
const MAX_TRACE = 1500;
|
||||
/** Values under this length sit two-per-row in the Slack attachment. */
|
||||
const SHORT_FIELD_LENGTH = 40;
|
||||
|
||||
const STYLE: Record<PagerSeverity, { emoji: string; color: string }> = {
|
||||
critical: { emoji: ':rotating_light:', color: '#d64545' },
|
||||
error: { emoji: ':red_circle:', color: '#e08d4c' },
|
||||
warning: { emoji: ':warning:', color: '#e0b84c' },
|
||||
info: { emoji: ':information_source:', color: '#4c8de0' },
|
||||
};
|
||||
|
||||
interface SlackField {
|
||||
title: string;
|
||||
value: string;
|
||||
short: boolean;
|
||||
}
|
||||
|
||||
export interface SlackMessage {
|
||||
text: string;
|
||||
channel?: string;
|
||||
username?: string;
|
||||
icon_emoji?: string;
|
||||
attachments: Array<{
|
||||
color: string;
|
||||
fallback: string;
|
||||
fields: SlackField[];
|
||||
footer: string;
|
||||
mrkdwn_in: string[];
|
||||
text?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
function truncate(value: string, max: number): string {
|
||||
return value.length <= max ? value : `${value.slice(0, max - 1)}…`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render an alert as an incoming-webhook payload: severity-coloured attachment,
|
||||
* the alarm's fields as a table, and the stack (when there is one) as a code
|
||||
* block. Split out from the handler so the formatting is testable on its own.
|
||||
*/
|
||||
export function buildSlackMessage(
|
||||
alert: AlertPayload,
|
||||
opts: { channel?: string; username?: string; serverId?: string } = {},
|
||||
): SlackMessage {
|
||||
const style = STYLE[alert.severity] ?? STYLE.info;
|
||||
const repeat = alert.isRepeat ? ` (x${alert.repeatCount})` : '';
|
||||
const headline = `${style.emoji} *[${alert.severity.toUpperCase()}]* ${alert.message}${repeat}`;
|
||||
|
||||
const fields: SlackField[] = [];
|
||||
for (const [key, value] of Object.entries(alert.fields)) {
|
||||
if (fields.length >= MAX_FIELDS) break;
|
||||
// `error` is already rendered as the trace block below.
|
||||
if (key === 'error') continue;
|
||||
const rendered = truncate(value, MAX_FIELD_VALUE);
|
||||
fields.push({
|
||||
title: key,
|
||||
value: rendered,
|
||||
short: rendered.length <= SHORT_FIELD_LENGTH,
|
||||
});
|
||||
}
|
||||
|
||||
const footerParts = [alert.shortId, alert.id];
|
||||
if (opts.serverId) footerParts.push(opts.serverId);
|
||||
|
||||
return {
|
||||
text: headline,
|
||||
...(opts.channel ? { channel: opts.channel } : {}),
|
||||
...(opts.username ? { username: opts.username } : {}),
|
||||
attachments: [
|
||||
{
|
||||
color: style.color,
|
||||
fallback: `[${alert.severity}] ${alert.message}`,
|
||||
fields,
|
||||
footer: footerParts.join(' • '),
|
||||
mrkdwn_in: ['text'],
|
||||
...(alert.trace
|
||||
? { text: '```' + truncate(alert.trace, MAX_TRACE) + '```' }
|
||||
: {}),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Post alerts to a Slack incoming webhook. Repeats of the same alarm id are
|
||||
* throttled so a hot loop doesn't flood the channel — the occurrence count on
|
||||
* the next post that gets through tells the reader what they missed.
|
||||
*/
|
||||
export function createSlackAlertHandler(
|
||||
conf: ISlackAlertConfig,
|
||||
opts: { serverId?: string } = {},
|
||||
): AlertHandler {
|
||||
const webhookUrl = conf.webhookUrl as string;
|
||||
const throttleMs = conf.repeatThrottleMs ?? DEFAULT_REPEAT_THROTTLE_MS;
|
||||
// A zero or negative throttle expires every entry on arrival.
|
||||
const lastPosted = new BoundedTtlMap<string, true>({
|
||||
maxEntries: MAX_THROTTLE_ENTRIES,
|
||||
ttlMs: throttleMs,
|
||||
});
|
||||
|
||||
return async (alert) => {
|
||||
if (!lastPosted.shouldEmit(alert.id)) return;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const res = await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(
|
||||
buildSlackMessage(alert, {
|
||||
channel: conf.channel,
|
||||
username: conf.username,
|
||||
serverId: opts.serverId,
|
||||
}),
|
||||
),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok) {
|
||||
// A rejected post shouldn't leave the id marked as delivered.
|
||||
lastPosted.delete(alert.id);
|
||||
throw new Error(`Slack webhook returned ${res.status}`);
|
||||
}
|
||||
} catch (err) {
|
||||
lastPosted.delete(alert.id);
|
||||
throw err;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { PagerSeverity } from '../../types';
|
||||
|
||||
export interface AlarmFields {
|
||||
error?: Error;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface AlarmOccurrence {
|
||||
message: string;
|
||||
fields: AlarmFields;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export interface AlarmOptions {
|
||||
/**
|
||||
* Collapse repeats of this alarm id into a single incident on transports
|
||||
* that de-duplicate (PagerDuty). Only right when the id already pins down
|
||||
* one recurring fault — an uncaught error on a route, say — so that N
|
||||
* occurrences really are one thing to fix. Off by default: every other
|
||||
* alarm raises its own incident per occurrence.
|
||||
*/
|
||||
dedup?: boolean;
|
||||
}
|
||||
|
||||
export interface Alarm extends AlarmOptions {
|
||||
id: string;
|
||||
shortId: string;
|
||||
message: string;
|
||||
fields: AlarmFields;
|
||||
error?: Error;
|
||||
started: number;
|
||||
/**
|
||||
* Every occurrence ever counted, including those aged out of the two lists
|
||||
* below.
|
||||
*/
|
||||
count: number;
|
||||
/**
|
||||
* Timestamps of the most recent occurrences only — see
|
||||
* `OCCURRENCE_HISTORY_LIMIT`.
|
||||
*/
|
||||
timestamps: number[];
|
||||
/** The most recent occurrences only — see `OCCURRENCE_HISTORY_LIMIT`. */
|
||||
occurrences: AlarmOccurrence[];
|
||||
severity?: PagerSeverity;
|
||||
noAlert?: boolean;
|
||||
/** Set once a config mute has been logged, so it's reported only once. */
|
||||
muteLogged?: boolean;
|
||||
}
|
||||
|
||||
export interface AlertPayload {
|
||||
id: string;
|
||||
/**
|
||||
* Key a de-duplicating transport groups by. Equal to `id` for alarms raised
|
||||
* with `dedup`, and unique per occurrence for everything else.
|
||||
*/
|
||||
dedupKey: string;
|
||||
/** Readable slug for the same alarm, for humans quoting it back. */
|
||||
shortId: string;
|
||||
message: string;
|
||||
source: string;
|
||||
severity: PagerSeverity;
|
||||
/** Field values rendered as strings, ready to display. */
|
||||
fields: Record<string, string>;
|
||||
/** Stack of the attached error, when the alarm carried one. */
|
||||
trace?: string;
|
||||
/** Total occurrences of this alarm so far, including this one. */
|
||||
repeatCount: number;
|
||||
/** False for the first occurrence of an alarm id. */
|
||||
isRepeat: boolean;
|
||||
/** PagerDuty `custom_details` payload. */
|
||||
custom?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type AlertHandler = (alert: AlertPayload) => Promise<void>;
|
||||
|
||||
export interface KnownErrorRule {
|
||||
match: {
|
||||
id: string;
|
||||
message?: string;
|
||||
fields?: Record<string, unknown>;
|
||||
};
|
||||
action: {
|
||||
type: 'no-alert' | 'severity';
|
||||
value?: PagerSeverity;
|
||||
};
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
// Optional analytics client, registered by an extension. Only the surface the
|
||||
// app-stats path consumes is declared; extend it rather than widening to `any`.
|
||||
|
||||
export interface ClickhouseQueryResult {
|
||||
json<T = Record<string, unknown>>(): Promise<T[]>;
|
||||
}
|
||||
|
||||
export interface ClickhouseQueryParams {
|
||||
query: string;
|
||||
query_params?: Record<string, unknown>;
|
||||
format?: string;
|
||||
}
|
||||
|
||||
export interface ClickhouseClient {
|
||||
query(params: ClickhouseQueryParams): Promise<ClickhouseQueryResult>;
|
||||
}
|
||||
@@ -1,362 +0,0 @@
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { IConfig } from '../../types';
|
||||
import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
|
||||
import { DatabaseClientFactory } from './index.js';
|
||||
import { MySQLDatabaseClient } from './MySQLDatabaseClient.js';
|
||||
import { PostgresDatabaseClient } from './PostgresDatabaseClient.js';
|
||||
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
|
||||
|
||||
const config = (engine: string): IConfig =>
|
||||
({ port: 0, extensions: [], database: { engine } }) as IConfig;
|
||||
|
||||
const sqlite = () => new SqliteDatabaseClient(config('sqlite'));
|
||||
const mysql = () => new MySQLDatabaseClient(config('mysql'));
|
||||
const postgres = () => new PostgresDatabaseClient(config('postgres'));
|
||||
|
||||
/**
|
||||
* Minimal concrete client backed by fixture rows — exercises the shared helpers
|
||||
* on `AbstractDatabaseClient` that the real engines inherit.
|
||||
*/
|
||||
class FixtureDatabaseClient extends AbstractDatabaseClient {
|
||||
override readonly engineName = 'fixture';
|
||||
readonly reads: string[] = [];
|
||||
readonly preads: string[] = [];
|
||||
readonly writes: { query: string; params: unknown[] }[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly replicaRows: Record<string, unknown>[] | Error,
|
||||
private readonly primaryRows: Record<string, unknown>[] = [],
|
||||
private readonly replica = true,
|
||||
) {
|
||||
super(config('fixture'));
|
||||
}
|
||||
|
||||
protected override hasReadReplica(): boolean {
|
||||
return this.replica;
|
||||
}
|
||||
|
||||
override async read(query: string): Promise<Record<string, unknown>[]> {
|
||||
this.reads.push(query);
|
||||
if (this.replicaRows instanceof Error) throw this.replicaRows;
|
||||
return this.replicaRows;
|
||||
}
|
||||
|
||||
override async pread(query: string): Promise<Record<string, unknown>[]> {
|
||||
this.preads.push(query);
|
||||
return this.primaryRows;
|
||||
}
|
||||
|
||||
override async write(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<WriteResult> {
|
||||
this.writes.push({ query, params });
|
||||
return { insertId: 1, affectedRows: 1, anyRowsAffected: true };
|
||||
}
|
||||
}
|
||||
|
||||
describe('DatabaseClientFactory', () => {
|
||||
it('picks sqlite when no engine is configured', () => {
|
||||
expect(
|
||||
new DatabaseClientFactory({ port: 0, extensions: [] } as IConfig),
|
||||
).toBeInstanceOf(SqliteDatabaseClient);
|
||||
});
|
||||
|
||||
it('refuses an unknown engine name', () => {
|
||||
expect(() => new DatabaseClientFactory(config('cassandra'))).toThrow(
|
||||
'Unknown database engine: cassandra',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — unimplemented surface', () => {
|
||||
const bare = () => new AbstractDatabaseClient(config('none'));
|
||||
|
||||
it('refuses to run queries until a subclass implements them', async () => {
|
||||
await expect(bare().read('SELECT 1')).rejects.toThrow(
|
||||
'DatabaseClient.read() not implemented',
|
||||
);
|
||||
await expect(bare().pread('SELECT 1')).rejects.toThrow(
|
||||
'DatabaseClient.pread() not implemented',
|
||||
);
|
||||
await expect(bare().write('DELETE FROM x')).rejects.toThrow(
|
||||
'DatabaseClient.write() not implemented',
|
||||
);
|
||||
await expect(bare().batchWrite([])).rejects.toThrow(
|
||||
'DatabaseClient.batchWrite() not implemented',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — replica-aware reads', () => {
|
||||
it('avoids a primary read when the replica has rows in sequential mode', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
expect(
|
||||
await client.tryHardRead('SELECT 1', [], { race: false }),
|
||||
).toEqual([{ id: 1 }]);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([[], new Error('replica unavailable')])(
|
||||
'falls back once in sequential mode',
|
||||
async (replica) => {
|
||||
const client = new FixtureDatabaseClient(replica, [{ id: 2 }]);
|
||||
expect(
|
||||
await client.tryHardRead('SELECT 1', [], { race: false }),
|
||||
).toEqual([{ id: 2 }]);
|
||||
expect(client.preads).toEqual(['SELECT 1']);
|
||||
},
|
||||
);
|
||||
|
||||
it('returns replica rows without waiting on the primary', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ id: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the primary when the replica has not caught up', async () => {
|
||||
const client = new FixtureDatabaseClient([], [{ id: 2 }]);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ id: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the primary when the replica read throws', async () => {
|
||||
const client = new FixtureDatabaseClient(
|
||||
new Error('replica unavailable'),
|
||||
[{ id: 3 }],
|
||||
);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ id: 3 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('surfaces the primary error when both reads fail', async () => {
|
||||
const client = new FixtureDatabaseClient(new Error('replica down'));
|
||||
client.pread = async () => {
|
||||
throw new Error('primary down');
|
||||
};
|
||||
await expect(client.tryHardRead('SELECT 1')).rejects.toThrow(
|
||||
'primary down',
|
||||
);
|
||||
});
|
||||
|
||||
it('issues a single read when there is no replica', async () => {
|
||||
const client = new FixtureDatabaseClient([], [{ id: 2 }], false);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([]);
|
||||
expect(client.reads).toEqual(['SELECT 1']);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
describe('tryHardReadAll', () => {
|
||||
const reads = [{ query: 'SELECT 1' }, { query: 'SELECT 2' }];
|
||||
|
||||
it('keeps replica rows without reading the primary', async () => {
|
||||
const client = new FixtureDatabaseClient([{ id: 1 }], [{ id: 2 }]);
|
||||
expect(await client.tryHardReadAll(reads)).toEqual([
|
||||
[{ id: 1 }],
|
||||
[{ id: 1 }],
|
||||
]);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([[], new Error('replica unavailable')])(
|
||||
'reads every query on the primary when the replica has none',
|
||||
async (replica) => {
|
||||
const client = new FixtureDatabaseClient(replica, [{ id: 2 }]);
|
||||
expect(await client.tryHardReadAll(reads)).toEqual([
|
||||
[{ id: 2 }],
|
||||
[{ id: 2 }],
|
||||
]);
|
||||
expect(client.preads).toEqual(['SELECT 1', 'SELECT 2']);
|
||||
},
|
||||
);
|
||||
|
||||
it('issues one read per query when there is no replica', async () => {
|
||||
const client = new FixtureDatabaseClient([], [{ id: 2 }], false);
|
||||
expect(await client.tryHardReadAll(reads)).toEqual([[], []]);
|
||||
expect(client.reads).toEqual(['SELECT 1', 'SELECT 2']);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — SQL generation', () => {
|
||||
it('generates an INSERT with quoted columns and placeholders', async () => {
|
||||
const client = new FixtureDatabaseClient([]);
|
||||
await client.insert('user', { username: 'ada', email: null });
|
||||
|
||||
expect(client.writes).toEqual([
|
||||
{
|
||||
query: 'INSERT INTO `user` (`username`, `email`) VALUES (?, ?)',
|
||||
params: ['ada', null],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('quotes each dotted segment and escapes embedded backticks', () => {
|
||||
const client = sqlite();
|
||||
expect(client.quoteIdentifier('user')).toBe('`user`');
|
||||
expect(client.quoteIdentifier('db.user.id')).toBe('`db`.`user`.`id`');
|
||||
expect(client.quoteIdentifier('user.*')).toBe('`user`.*');
|
||||
expect(client.quoteIdentifier('we`ird')).toBe('`we``ird`');
|
||||
});
|
||||
|
||||
it('quotes postgres identifiers with double quotes', () => {
|
||||
const client = postgres();
|
||||
expect(client.quoteIdentifier('db.user')).toBe('"db"."user"');
|
||||
expect(client.quoteIdentifier('we"ird')).toBe('"we""ird"');
|
||||
expect(client.quoteIdentifier('user.*')).toBe('"user".*');
|
||||
});
|
||||
|
||||
it('renders booleans the way each engine expects', () => {
|
||||
expect(sqlite().booleanLiteral(true)).toBe('1');
|
||||
expect(sqlite().booleanValue(false)).toBe(0);
|
||||
expect(postgres().booleanLiteral(true)).toBe('TRUE');
|
||||
expect(postgres().booleanLiteral(false)).toBe('FALSE');
|
||||
expect(postgres().booleanValue(true)).toBe(true);
|
||||
});
|
||||
|
||||
it('picks the engine-specific ignore-conflict syntax', () => {
|
||||
expect(sqlite().insertIgnoreInto('kv')).toBe(
|
||||
'INSERT OR IGNORE INTO `kv`',
|
||||
);
|
||||
expect(sqlite().insertIgnoreSuffix()).toBe('');
|
||||
expect(postgres().insertIgnoreInto('kv')).toBe('INSERT INTO "kv"');
|
||||
expect(postgres().insertIgnoreSuffix()).toBe(' ON CONFLICT DO NOTHING');
|
||||
expect(mysql().insertIgnoreInto('kv')).toBe('INSERT IGNORE INTO `kv`');
|
||||
expect(mysql().insertIgnoreSuffix()).toBe('');
|
||||
});
|
||||
|
||||
it('builds an upsert clause per engine', () => {
|
||||
expect(mysql().upsertClause(['user_id'], ['value', 'dt'])).toBe(
|
||||
'ON DUPLICATE KEY UPDATE `value` = ?, `dt` = ?',
|
||||
);
|
||||
expect(sqlite().upsertClause(['user_id', 'app'], ['value'])).toBe(
|
||||
'ON CONFLICT(`user_id`, `app`) DO UPDATE SET `value` = ?',
|
||||
);
|
||||
expect(postgres().upsertClause(['user_id'], ['value'])).toBe(
|
||||
'ON CONFLICT("user_id") DO UPDATE SET "value" = ?',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects an upsert with nothing to update', () => {
|
||||
expect(() => sqlite().upsertClause(['user_id'], [])).toThrow(
|
||||
'upsertClause requires at least one update column',
|
||||
);
|
||||
});
|
||||
|
||||
it('extracts JSON text using each engine dialect', () => {
|
||||
expect(sqlite().jsonTextExtract('`metadata`', ['a', 'b'])).toBe(
|
||||
"json_extract(`metadata`, '$.a.b')",
|
||||
);
|
||||
expect(mysql().jsonTextExtract('`metadata`', ['a'])).toBe(
|
||||
"JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.a'))",
|
||||
);
|
||||
expect(postgres().jsonTextExtract('"metadata"', ['a', 'b'])).toBe(
|
||||
`"metadata" #>> ARRAY['a', 'b']`,
|
||||
);
|
||||
});
|
||||
|
||||
it('escapes quotes inside JSON path segments', () => {
|
||||
expect(sqlite().jsonTextExtract('`m`', ["it's"])).toBe(
|
||||
"json_extract(`m`, '$.it''s')",
|
||||
);
|
||||
});
|
||||
|
||||
it('coalesces expressions and rejects an empty list', () => {
|
||||
expect(sqlite().nullCoalesce('`a`', '`b`', '0')).toBe(
|
||||
'COALESCE(`a`, `b`, 0)',
|
||||
);
|
||||
expect(() => sqlite().nullCoalesce()).toThrow(
|
||||
'nullCoalesce requires at least one expression',
|
||||
);
|
||||
});
|
||||
|
||||
it('only postgres needs a RETURNING clause to learn the insert id', () => {
|
||||
expect(postgres().returningIdClause()).toBe(' RETURNING id');
|
||||
expect(sqlite().returningIdClause()).toBe('');
|
||||
expect(mysql().returningIdClause()).toBe('');
|
||||
});
|
||||
|
||||
it('stamps a timestamp from unix seconds in each dialect', () => {
|
||||
expect(sqlite().timestampFromUnixSql()).toBe(
|
||||
"datetime(?, 'unixepoch')",
|
||||
);
|
||||
expect(mysql().timestampFromUnixSql()).toBe('FROM_UNIXTIME(?)');
|
||||
expect(postgres().timestampFromUnixSql()).toBe('to_timestamp(?)');
|
||||
});
|
||||
|
||||
it('reads a timestamp column back as unix seconds in each engine', () => {
|
||||
expect(sqlite().unixSecondsSql('`t`')).toBe(
|
||||
"CAST(strftime('%s', `t`) AS INTEGER)",
|
||||
);
|
||||
expect(mysql().unixSecondsSql('`t`')).toBe('UNIX_TIMESTAMP(`t`)');
|
||||
expect(postgres().unixSecondsSql('`t`')).toBe(
|
||||
'EXTRACT(EPOCH FROM `t`)::bigint',
|
||||
);
|
||||
});
|
||||
|
||||
it('reads the database clock as epoch milliseconds in each dialect', () => {
|
||||
expect(sqlite().nowMsExpr()).toBe(
|
||||
"CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
|
||||
);
|
||||
expect(mysql().nowMsExpr()).toBe(
|
||||
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
|
||||
);
|
||||
expect(postgres().nowMsExpr()).toBe(
|
||||
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
|
||||
);
|
||||
});
|
||||
|
||||
it('describes a stored timestamp the way each driver reads it back', () => {
|
||||
const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
|
||||
expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');
|
||||
expect(mysql().timestampValue(seconds)).toEqual(
|
||||
new Date(seconds * 1000),
|
||||
);
|
||||
expect(postgres().timestampValue(seconds)).toEqual(
|
||||
new Date(seconds * 1000),
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to `otherwise` for an engine with no explicit choice', () => {
|
||||
const client = new FixtureDatabaseClient([]);
|
||||
expect(
|
||||
client.case({ sqlite: 'a', mysql: 'b', otherwise: 'fallback' }),
|
||||
).toBe('fallback');
|
||||
expect(client.case({ fixture: 'exact', otherwise: 'fallback' })).toBe(
|
||||
'exact',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
it('builds an insert for batchWrite and shares it with insert', async () => {
|
||||
const client = new FixtureDatabaseClient([]);
|
||||
const entry = client.insertEntry('widget', { name: "reader's", enabled: true });
|
||||
expect(entry.values).toEqual(["reader's", true]);
|
||||
expect(entry.statement).toContain('VALUES (?, ?)');
|
||||
await client.insert('widget', { name: "reader's", enabled: true });
|
||||
expect(client.writes).toEqual([{ query: entry.statement, params: entry.values }]);
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
/*
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
@@ -17,23 +17,13 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { isAbsolute, resolve as resolvePath } from 'path';
|
||||
import type { IConfig } from '../../types';
|
||||
import { Span } from '../../util/span.js';
|
||||
import { PuterClient } from '../types';
|
||||
import { compareMigrationFilenames } from './migrationFilenames.js';
|
||||
|
||||
export interface WriteResult {
|
||||
insertId: number | bigint;
|
||||
affectedRows: number;
|
||||
anyRowsAffected: boolean;
|
||||
/**
|
||||
* Warnings the statement raised; nonzero means a value may have been stored
|
||||
* differently from how it was sent. Absent when the engine can't vouch that
|
||||
* a clean insert stored exactly what was sent, on a fresh id.
|
||||
*/
|
||||
warningStatus?: number;
|
||||
}
|
||||
|
||||
export interface BatchEntry {
|
||||
@@ -41,20 +31,12 @@ export interface BatchEntry {
|
||||
values: unknown[];
|
||||
}
|
||||
|
||||
type SqlJsonPath = readonly [string, ...string[]];
|
||||
|
||||
/** The slice of a pooled connection that migrations need. */
|
||||
export interface MigrationConnection {
|
||||
query(sql: string): Promise<unknown>;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base database client. Subclasses must override every method that throws here.
|
||||
*
|
||||
* Do not instantiate directly — use the factory exported from
|
||||
* `clients/database/index.ts` which picks the right implementation based on
|
||||
* `config.database.engine`.
|
||||
* `clients/database/index.ts` which picks the right implementation
|
||||
* based on `config.database.engine`.
|
||||
*/
|
||||
export class AbstractDatabaseClient extends PuterClient {
|
||||
/** Short name used by `case()` to pick engine-specific values. */
|
||||
@@ -68,7 +50,9 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
// Abstract interface — subclasses MUST override
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/** Execute a read query. Returns an array of row objects. */
|
||||
/**
|
||||
* Execute a read query. Returns an array of row objects.
|
||||
*/
|
||||
async read(
|
||||
_query: string,
|
||||
_params: unknown[] = [],
|
||||
@@ -77,9 +61,9 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read that prefers the primary database (useful when read-replicas may
|
||||
* have replication lag). In single-node setups this is identical to
|
||||
* `read()`.
|
||||
* Read that prefers the primary database (useful when read-replicas
|
||||
* may have replication lag). In single-node setups this is identical
|
||||
* to `read()`.
|
||||
*/
|
||||
async pread(
|
||||
_query: string,
|
||||
@@ -88,12 +72,16 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
throw new Error('DatabaseClient.pread() not implemented');
|
||||
}
|
||||
|
||||
/** Execute a write query (INSERT / UPDATE / DELETE). */
|
||||
/**
|
||||
* Execute a write query (INSERT / UPDATE / DELETE).
|
||||
*/
|
||||
async write(_query: string, _params: unknown[] = []): Promise<WriteResult> {
|
||||
throw new Error('DatabaseClient.write() not implemented');
|
||||
}
|
||||
|
||||
/** Execute multiple write statements in a single transaction. */
|
||||
/**
|
||||
* Execute multiple write statements in a single transaction.
|
||||
*/
|
||||
async batchWrite(_entries: BatchEntry[]): Promise<void> {
|
||||
throw new Error('DatabaseClient.batchWrite() not implemented');
|
||||
}
|
||||
@@ -110,194 +98,59 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
tableName: string,
|
||||
data: Record<string, unknown>,
|
||||
): Promise<WriteResult> {
|
||||
const entry = this.insertEntry(tableName, data);
|
||||
return this.write(
|
||||
entry.statement + this.returningIdClause(),
|
||||
entry.values,
|
||||
);
|
||||
}
|
||||
|
||||
/** An insert that can join other writes in a transaction. */
|
||||
insertEntry(tableName: string, data: Record<string, unknown>): BatchEntry {
|
||||
const cols = Object.keys(data);
|
||||
return {
|
||||
statement:
|
||||
`INSERT INTO ${this.quoteIdentifier(tableName)} ` +
|
||||
`(${cols.map((c) => this.quoteIdentifier(c)).join(', ')}) ` +
|
||||
`VALUES (${cols.map(() => '?').join(', ')})`,
|
||||
values: Object.values(data),
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether `read()` goes to a separate replica rather than the primary. */
|
||||
protected hasReadReplica(): boolean {
|
||||
return false;
|
||||
const values = Object.values(data);
|
||||
const sql =
|
||||
`INSERT INTO \`${tableName}\` ` +
|
||||
`(${cols.map((c) => `\`${c}\``).join(', ')}) ` +
|
||||
`VALUES (${cols.map(() => '?').join(', ')})`;
|
||||
return this.write(sql, values);
|
||||
}
|
||||
|
||||
/**
|
||||
* The replica leg of `tryHardRead`. Override when `read()` falls back to
|
||||
* the primary by itself, or a replica failure reads the primary twice.
|
||||
* Like `read()` but falls back to the primary when read-replicas are
|
||||
* in use. Subclasses may override with replica-aware logic; the
|
||||
* default delegates to `pread()`.
|
||||
*/
|
||||
protected replicaRead(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Like `read()`, but an empty or failed replica read falls back to the
|
||||
* primary, so replication lag can't hide a row that was just written. By
|
||||
* default the primary read starts alongside the replica one; `race: false`
|
||||
* reads the primary only after the replica returns empty or fails. Without
|
||||
* a replica this is a single `read()`.
|
||||
*/
|
||||
@Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
|
||||
async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
{ race = true }: { race?: boolean } = {},
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (!this.hasReadReplica()) return this.read(query, params);
|
||||
const primary = this.pread(query, params);
|
||||
primary.catch(() => {});
|
||||
|
||||
const primary = race ? settle(this.pread(query, params)) : null;
|
||||
try {
|
||||
const rows = await this.replicaRead(query, params);
|
||||
if (rows.length > 0) return rows;
|
||||
const rows = await this.read(query, params);
|
||||
if (rows.length > 0) {
|
||||
return rows;
|
||||
}
|
||||
} catch {
|
||||
// replica failed — fall through to primary
|
||||
}
|
||||
const result = await (primary ?? settle(this.pread(query, params)));
|
||||
if ('error' in result) throw result.error;
|
||||
return result.value;
|
||||
}
|
||||
|
||||
/** Reads the primary only when every replica read is empty or fails. */
|
||||
@Span('db.tryHardReadAll', (reads: unknown[]) => ({
|
||||
'db.batch_size': reads.length,
|
||||
}))
|
||||
async tryHardReadAll(
|
||||
reads: { query: string; params?: unknown[] }[],
|
||||
): Promise<Record<string, unknown>[][]> {
|
||||
if (!this.hasReadReplica()) {
|
||||
return Promise.all(
|
||||
reads.map(({ query, params = [] }) => this.read(query, params)),
|
||||
);
|
||||
}
|
||||
const onReplica = await Promise.all(
|
||||
reads.map(async ({ query, params = [] }) => {
|
||||
try {
|
||||
return await this.replicaRead(query, params);
|
||||
} catch {
|
||||
// A failed replica read counts as a miss.
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (onReplica.some((rows) => rows.length > 0)) return onReplica;
|
||||
return Promise.all(
|
||||
reads.map(({ query, params = [] }) => this.pread(query, params)),
|
||||
);
|
||||
return primary;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the `.sql` files named `<engineName>*` in each configured migration
|
||||
* directory, in filename order. Files must be idempotent: nothing records
|
||||
* which ones already ran. A failure aborts startup. `transactional` wraps
|
||||
* each file in BEGIN/COMMIT, for engines whose DDL can roll back.
|
||||
* Like `tryHardRead()` but throws when the result set is empty.
|
||||
*/
|
||||
protected async applyMigrationPaths(
|
||||
connect: () => Promise<MigrationConnection>,
|
||||
splitStatements: (contents: string) => string[],
|
||||
{ transactional }: { transactional: boolean },
|
||||
): Promise<void> {
|
||||
const paths = this.config.database?.migrationPaths;
|
||||
if (!paths || paths.length === 0) return;
|
||||
|
||||
const tag = `[${this.engineName}]`;
|
||||
const conn = await connect();
|
||||
try {
|
||||
for (const rawPath of paths) {
|
||||
const dir = isAbsolute(rawPath)
|
||||
? rawPath
|
||||
: resolvePath(process.cwd(), rawPath);
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = readdirSync(dir)
|
||||
.filter(
|
||||
(f) =>
|
||||
f.endsWith('.sql') &&
|
||||
f.startsWith(this.engineName),
|
||||
)
|
||||
.sort(compareMigrationFilenames);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`${tag} migration path is unreadable: ${dir}`,
|
||||
{
|
||||
cause: e,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log(`${tag} no migrations in ${dir}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${tag} running migrations from ${dir}: ${files.length} file(s)`,
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
const contents = readFileSync(
|
||||
resolvePath(dir, file),
|
||||
'utf8',
|
||||
);
|
||||
const statements = splitStatements(contents);
|
||||
if (transactional) await conn.query('BEGIN');
|
||||
try {
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
try {
|
||||
await conn.query(statements[i]);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`${tag} failed to apply ${file} at statement ${i}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
}
|
||||
if (transactional) await conn.query('COMMIT');
|
||||
} catch (e) {
|
||||
if (transactional) {
|
||||
try {
|
||||
await conn.query('ROLLBACK');
|
||||
} catch {
|
||||
// the original failure is the one worth surfacing
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
console.log(
|
||||
`${tag} applied ${file} (${statements.length} statements)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
async requireRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const rows = await this.tryHardRead(query, params);
|
||||
if (rows.length === 0) {
|
||||
throw new Error(`required read returned no rows: ${query}`);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the value from `choices` that matches the current engine.
|
||||
*
|
||||
* Usage:
|
||||
*
|
||||
* db.case({
|
||||
* sqlite: "datetime('now')",
|
||||
* mysql: 'NOW()',
|
||||
* otherwise: 'NOW()',
|
||||
* });
|
||||
* ```
|
||||
* db.case({ sqlite: "datetime('now')", mysql: 'NOW()', otherwise: 'NOW()' })
|
||||
* ```
|
||||
*
|
||||
* If the engine name isn't present in `choices`, falls back to
|
||||
* `choices.otherwise`.
|
||||
@@ -308,176 +161,4 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
}
|
||||
return choices.otherwise as T;
|
||||
}
|
||||
|
||||
quoteIdentifier(identifier: string): string {
|
||||
return identifier
|
||||
.split('.')
|
||||
.map((part) => {
|
||||
if (part === '*') return part;
|
||||
return `\`${part.replaceAll('`', '``')}\``;
|
||||
})
|
||||
.join('.');
|
||||
}
|
||||
|
||||
booleanLiteral(value: boolean): string {
|
||||
return value ? '1' : '0';
|
||||
}
|
||||
|
||||
booleanValue(value: boolean): boolean | 0 | 1 {
|
||||
return value ? 1 : 0;
|
||||
}
|
||||
|
||||
insertIgnoreInto(tableName: string): string {
|
||||
const table = this.quoteIdentifier(tableName);
|
||||
return this.case({
|
||||
sqlite: `INSERT OR IGNORE INTO ${table}`,
|
||||
postgres: `INSERT INTO ${table}`,
|
||||
otherwise: `INSERT IGNORE INTO ${table}`,
|
||||
});
|
||||
}
|
||||
|
||||
insertIgnoreSuffix(): string {
|
||||
return this.case({
|
||||
postgres: ' ON CONFLICT DO NOTHING',
|
||||
otherwise: '',
|
||||
});
|
||||
}
|
||||
|
||||
upsertClause(
|
||||
conflictColumns: readonly string[],
|
||||
updateColumns: readonly string[],
|
||||
): string {
|
||||
if (updateColumns.length === 0) {
|
||||
throw new Error('upsertClause requires at least one update column');
|
||||
}
|
||||
|
||||
const updateList = updateColumns
|
||||
.map((column) => `${this.quoteIdentifier(column)} = ?`)
|
||||
.join(', ');
|
||||
|
||||
return this.case({
|
||||
mysql: `ON DUPLICATE KEY UPDATE ${updateList}`,
|
||||
otherwise: `ON CONFLICT(${conflictColumns
|
||||
.map((column) => this.quoteIdentifier(column))
|
||||
.join(', ')}) DO UPDATE SET ${updateList}`,
|
||||
});
|
||||
}
|
||||
|
||||
jsonTextExtract(jsonExpression: string, path: SqlJsonPath): string {
|
||||
const sqlitePath = `$${path.map((part) => `.${part}`).join('')}`;
|
||||
return this.case({
|
||||
sqlite: `json_extract(${jsonExpression}, ${this.sqlStringLiteral(sqlitePath)})`,
|
||||
mysql: `JSON_UNQUOTE(JSON_EXTRACT(${jsonExpression}, ${this.sqlStringLiteral(sqlitePath)}))`,
|
||||
postgres: `${jsonExpression} #>> ARRAY[${path
|
||||
.map((part) => this.sqlStringLiteral(part))
|
||||
.join(', ')}]`,
|
||||
otherwise: `JSON_UNQUOTE(JSON_EXTRACT(${jsonExpression}, ${this.sqlStringLiteral(sqlitePath)}))`,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Whole `periodSeconds` buckets since the epoch, from a unix-seconds
|
||||
* column. MySQL's `/` yields a decimal, which would split a bucket.
|
||||
*/
|
||||
periodBucket(secondsExpression: string, periodSeconds: number): string {
|
||||
return this.case({
|
||||
mysql: `(${secondsExpression} DIV ${periodSeconds})`,
|
||||
otherwise: `(${secondsExpression} / ${periodSeconds})`,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts each (`userExpression`, bucket) pair once. `COUNT(DISTINCT a, b)`
|
||||
* is MySQL-only, so the pair folds into one number — injective while the
|
||||
* bucket stays under the multiplier, which hourly reaches in the 2980s.
|
||||
*/
|
||||
countDistinctUserPeriods(
|
||||
userExpression: string,
|
||||
secondsExpression: string,
|
||||
periodSeconds: number,
|
||||
): string {
|
||||
return `COUNT(DISTINCT ${userExpression} * 10000000 + ${this.periodBucket(secondsExpression, periodSeconds)})`;
|
||||
}
|
||||
|
||||
nullCoalesce(...expressions: readonly string[]): string {
|
||||
if (expressions.length === 0) {
|
||||
throw new Error('nullCoalesce requires at least one expression');
|
||||
}
|
||||
return `COALESCE(${expressions.join(', ')})`;
|
||||
}
|
||||
|
||||
returningIdClause(): string {
|
||||
return this.case({
|
||||
postgres: ' RETURNING id',
|
||||
otherwise: '',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A timestamp column value from one bound parameter in unix seconds, stored
|
||||
* the way `CURRENT_TIMESTAMP` would store that instant.
|
||||
*/
|
||||
timestampFromUnixSql(): string {
|
||||
return this.case({
|
||||
sqlite: "datetime(?, 'unixepoch')",
|
||||
postgres: 'to_timestamp(?)',
|
||||
otherwise: 'FROM_UNIXTIME(?)',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A timestamp column as unix seconds, computed by the database: the mysql
|
||||
* driver reads a stored UTC timestamp as local time, so parsing it in JS
|
||||
* can be off by the server's UTC offset.
|
||||
*/
|
||||
unixSecondsSql(column: string): string {
|
||||
return this.case({
|
||||
postgres: `EXTRACT(EPOCH FROM ${column})::bigint`,
|
||||
mysql: `UNIX_TIMESTAMP(${column})`,
|
||||
otherwise: `CAST(strftime('%s', ${column}) AS INTEGER)`,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The database's own clock as epoch milliseconds, so nodes whose clocks
|
||||
* disagree still agree on it. The mysql form ignores the session time zone,
|
||||
* so a DST change can't shift it.
|
||||
*/
|
||||
nowMsExpr(): string {
|
||||
return this.case({
|
||||
sqlite: "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)",
|
||||
postgres:
|
||||
'CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)',
|
||||
otherwise:
|
||||
"(TIMESTAMPDIFF(MICROSECOND, '1970-01-01 00:00:00', UTC_TIMESTAMP(6)) DIV 1000)",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* What a read of a timestamp column holding `seconds` returns: sqlite gives
|
||||
* back the UTC text it stored, mysql2 and pg a `Date` read in node's time
|
||||
* zone (so this matches when node and the database session share one).
|
||||
*/
|
||||
timestampValue(seconds: number): Date | string {
|
||||
const date = new Date(seconds * 1000);
|
||||
return this.case<Date | string>({
|
||||
sqlite: date.toISOString().slice(0, 19).replace('T', ' '),
|
||||
otherwise: date,
|
||||
});
|
||||
}
|
||||
|
||||
protected sqlStringLiteral(value: string): string {
|
||||
return `'${value.replaceAll("'", "''")}'`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Await later without an unhandled rejection in the meantime. */
|
||||
const settle = async <T>(
|
||||
promise: Promise<T>,
|
||||
): Promise<{ value: T } | { error: unknown }> => {
|
||||
try {
|
||||
return { value: await promise };
|
||||
} catch (error) {
|
||||
return { error };
|
||||
}
|
||||
};
|
||||
@@ -1,872 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { IConfig } from '../../types';
|
||||
import {
|
||||
MySQLDatabaseClient,
|
||||
compareMigrationFilenames,
|
||||
} from './MySQLDatabaseClient.js';
|
||||
|
||||
interface FakeConnection {
|
||||
query: ReturnType<typeof vi.fn>;
|
||||
execute: ReturnType<typeof vi.fn>;
|
||||
beginTransaction: ReturnType<typeof vi.fn>;
|
||||
commit: ReturnType<typeof vi.fn>;
|
||||
rollback: ReturnType<typeof vi.fn>;
|
||||
release: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
interface FakePool {
|
||||
poolConfig: Record<string, unknown>;
|
||||
connectionHandlers: ((conn: unknown) => void)[];
|
||||
connections: FakeConnection[];
|
||||
calls: { sql: string; values?: unknown[] }[];
|
||||
ended: boolean;
|
||||
endError: unknown;
|
||||
endThrows: unknown;
|
||||
respond: (sql: string, values?: unknown[]) => unknown;
|
||||
on: (event: string, handler: (conn: unknown) => void) => FakePool;
|
||||
promise: () => { getConnection: () => Promise<FakeConnection> };
|
||||
end: (cb: (err?: unknown) => void) => void;
|
||||
}
|
||||
|
||||
// mysql2 is the wire boundary: stub the driver's pool so we can assert on
|
||||
// the exact SQL, parameters and connection choreography the adapter emits.
|
||||
const { createPoolMock, createdPools } = vi.hoisted(() => {
|
||||
const createdPools: FakePool[] = [];
|
||||
|
||||
const createPoolMock = vi.fn((poolConfig: Record<string, unknown>) => {
|
||||
const pool: FakePool = {
|
||||
poolConfig,
|
||||
connectionHandlers: [],
|
||||
connections: [],
|
||||
calls: [],
|
||||
ended: false,
|
||||
endError: null,
|
||||
endThrows: null,
|
||||
respond: () => [[], undefined],
|
||||
on(event, handler) {
|
||||
if (event === 'connection') {
|
||||
pool.connectionHandlers.push(handler);
|
||||
}
|
||||
return pool;
|
||||
},
|
||||
promise() {
|
||||
return {
|
||||
getConnection: async () => {
|
||||
const run = async (sql: string, values?: unknown[]) => {
|
||||
pool.calls.push({ sql, values });
|
||||
return pool.respond(sql, values);
|
||||
};
|
||||
const conn: FakeConnection = {
|
||||
query: vi.fn(run),
|
||||
execute: vi.fn(run),
|
||||
beginTransaction: vi.fn(async () => {}),
|
||||
commit: vi.fn(async () => {}),
|
||||
rollback: vi.fn(async () => {}),
|
||||
release: vi.fn(),
|
||||
};
|
||||
pool.connections.push(conn);
|
||||
return conn;
|
||||
},
|
||||
};
|
||||
},
|
||||
end(cb) {
|
||||
if (pool.endThrows) throw pool.endThrows;
|
||||
pool.ended = true;
|
||||
cb(pool.endError ?? undefined);
|
||||
},
|
||||
};
|
||||
createdPools.push(pool);
|
||||
return pool;
|
||||
});
|
||||
|
||||
return { createPoolMock, createdPools };
|
||||
});
|
||||
|
||||
vi.mock('mysql2', () => ({ createPool: createPoolMock }));
|
||||
|
||||
describe('compareMigrationFilenames', () => {
|
||||
it('orders numbered migrations numerically, not lexically', () => {
|
||||
const input = [
|
||||
'mysql_mig_10.sql',
|
||||
'mysql_mig_2.sql',
|
||||
'mysql_mig_1.sql',
|
||||
'mysql_mig_9.sql',
|
||||
'mysql_mig_3.sql',
|
||||
];
|
||||
const sorted = [...input].sort(compareMigrationFilenames);
|
||||
expect(sorted).toEqual([
|
||||
'mysql_mig_1.sql',
|
||||
'mysql_mig_2.sql',
|
||||
'mysql_mig_3.sql',
|
||||
'mysql_mig_9.sql',
|
||||
'mysql_mig_10.sql',
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps mig_10 after mig_9 when the real prod set is shuffled', () => {
|
||||
// Reflects the current migrations/mysql/ listing — guards against
|
||||
// a future "let's just rename to padded" suggestion accidentally
|
||||
// re-introducing the lex bug if the rename is incomplete.
|
||||
const real = [
|
||||
'mysql_mig_9.sql',
|
||||
'mysql_mig_3.sql',
|
||||
'mysql_mig_10.sql',
|
||||
'mysql_mig_1.sql',
|
||||
'mysql_mig_7.sql',
|
||||
'mysql_mig_5.sql',
|
||||
'mysql_mig_2.sql',
|
||||
'mysql_mig_4.sql',
|
||||
'mysql_mig_8.sql',
|
||||
'mysql_mig_6.sql',
|
||||
];
|
||||
const sorted = [...real].sort(compareMigrationFilenames);
|
||||
for (let i = 1; i <= sorted.length; i += 1) {
|
||||
expect(sorted[i - 1]).toBe(`mysql_mig_${i}.sql`);
|
||||
}
|
||||
});
|
||||
|
||||
it('sorts non-numeric filenames after numbered ones, lexically among themselves', () => {
|
||||
// Numbered files always run first (they're the canonical history);
|
||||
// unmatched names follow in localeCompare order. Mixing the two
|
||||
// sets prevents a vendor dump from accidentally wedging itself
|
||||
// between mig_4 and mig_5 if it happened to lex-sort there.
|
||||
const mixed = [
|
||||
'mysql_mig_2.sql',
|
||||
'mysql_vendor_dump.sql',
|
||||
'mysql_mig_10.sql',
|
||||
'mysql_bootstrap.sql',
|
||||
'mysql_mig_1.sql',
|
||||
];
|
||||
const sorted = [...mixed].sort(compareMigrationFilenames);
|
||||
expect(sorted).toEqual([
|
||||
'mysql_mig_1.sql',
|
||||
'mysql_mig_2.sql',
|
||||
'mysql_mig_10.sql',
|
||||
'mysql_bootstrap.sql',
|
||||
'mysql_vendor_dump.sql',
|
||||
]);
|
||||
});
|
||||
|
||||
it('is stable for already-sorted input', () => {
|
||||
const sorted = [
|
||||
'mysql_mig_1.sql',
|
||||
'mysql_mig_2.sql',
|
||||
'mysql_mig_10.sql',
|
||||
'mysql_mig_11.sql',
|
||||
];
|
||||
expect([...sorted].sort(compareMigrationFilenames)).toEqual(sorted);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Replica read failover ───────────────────────────────────────────
|
||||
//
|
||||
// `read()` normally goes to the replica batcher; when the replica side is
|
||||
// degraded (batcher load-shed or a transient connection error) and a real
|
||||
// replica is configured, the read retries once on the primary read batcher.
|
||||
// The write batcher is left unset, so a failover that reached it would throw.
|
||||
|
||||
type Batcher = { execute: ReturnType<typeof vi.fn> };
|
||||
|
||||
const makeClient = (opts: {
|
||||
replica: Batcher;
|
||||
primary: Batcher;
|
||||
multiNode?: boolean;
|
||||
}) => {
|
||||
const client = new MySQLDatabaseClient({
|
||||
database: { engine: 'mysql' },
|
||||
} as IConfig);
|
||||
// Bypass onServerStart (which would connect to a real database) and
|
||||
// inject the batchers directly. Configuration enum: SINGLE=0, REPLICA=1.
|
||||
Object.assign(client as unknown as Record<string, unknown>, {
|
||||
dbReplica: opts.replica,
|
||||
dbPrimaryRead: opts.primary,
|
||||
configuration: opts.multiNode === false ? 0 : 1,
|
||||
});
|
||||
return client;
|
||||
};
|
||||
|
||||
const codedError = (code: string) => {
|
||||
const err = new Error(code) as Error & { code: string };
|
||||
err.code = code;
|
||||
return err;
|
||||
};
|
||||
|
||||
describe('MySQLDatabaseClient.read — replica failover', () => {
|
||||
it('fails over to the primary on batcher load-shed errors', async () => {
|
||||
const replica = {
|
||||
execute: vi.fn().mockRejectedValue(codedError('dbBatchFailed')),
|
||||
};
|
||||
const primary = { execute: vi.fn().mockResolvedValue([[{ ok: 1 }]]) };
|
||||
const client = makeClient({ replica, primary });
|
||||
|
||||
await expect(client.read('SELECT 1')).resolves.toEqual([{ ok: 1 }]);
|
||||
expect(primary.execute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('fails over on transient connection errors', async () => {
|
||||
const replica = {
|
||||
execute: vi.fn().mockRejectedValue(codedError('ECONNRESET')),
|
||||
};
|
||||
const primary = { execute: vi.fn().mockResolvedValue([[{ ok: 1 }]]) };
|
||||
const client = makeClient({ replica, primary });
|
||||
|
||||
await expect(client.read('SELECT 1')).resolves.toEqual([{ ok: 1 }]);
|
||||
});
|
||||
|
||||
it('rethrows deterministic SQL errors without touching the primary', async () => {
|
||||
const replica = {
|
||||
execute: vi.fn().mockRejectedValue(codedError('ER_PARSE_ERROR')),
|
||||
};
|
||||
const primary = { execute: vi.fn() };
|
||||
const client = makeClient({ replica, primary });
|
||||
|
||||
await expect(client.read('SELEC oops')).rejects.toMatchObject({
|
||||
code: 'ER_PARSE_ERROR',
|
||||
});
|
||||
expect(primary.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not fail over in single-node configuration', async () => {
|
||||
const replica = {
|
||||
execute: vi.fn().mockRejectedValue(codedError('dbBatchFailed')),
|
||||
};
|
||||
const primary = { execute: vi.fn() };
|
||||
const client = makeClient({ replica, primary, multiNode: false });
|
||||
|
||||
await expect(client.read('SELECT 1')).rejects.toMatchObject({
|
||||
code: 'dbBatchFailed',
|
||||
});
|
||||
expect(primary.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// -- Driver-level behaviour (stubbed mysql2 pool) ---------------------
|
||||
|
||||
const mysqlConfig = (
|
||||
database: Partial<NonNullable<IConfig['database']>> = {},
|
||||
): IConfig =>
|
||||
({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
database: { engine: 'mysql', ...database },
|
||||
}) as IConfig;
|
||||
|
||||
const startClient = async (
|
||||
database: Partial<NonNullable<IConfig['database']>> = {},
|
||||
): Promise<MySQLDatabaseClient> => {
|
||||
const client = new MySQLDatabaseClient(mysqlConfig(database));
|
||||
await client.onServerStart();
|
||||
return client;
|
||||
};
|
||||
|
||||
// The batcher coalesces a batch into `stmt1;stmt2; SELECT 1`, so the driver
|
||||
// must answer with one row-set per statement plus one for the SELECT 1.
|
||||
const rowSets = (...sets: unknown[][]) => [[...sets, [{ 1: 1 }]], undefined];
|
||||
|
||||
// Writes go out one statement at a time, so the driver answers with just the
|
||||
// result header.
|
||||
const writeHeader = (header: Record<string, unknown>) => [header, undefined];
|
||||
|
||||
beforeEach(() => {
|
||||
createdPools.length = 0;
|
||||
createPoolMock.mockClear();
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — pool construction', () => {
|
||||
it('builds the primary pool from config with multi-statement support', async () => {
|
||||
await startClient({
|
||||
host: 'db.internal',
|
||||
port: 3307,
|
||||
user: 'puter',
|
||||
password: 'hunter2',
|
||||
database: 'puterdb',
|
||||
});
|
||||
|
||||
expect(createPoolMock).toHaveBeenCalledTimes(1);
|
||||
expect(createdPools[0].poolConfig).toEqual({
|
||||
maxPreparedStatements: 900,
|
||||
connectionLimit: 30,
|
||||
maxIdle: 29,
|
||||
idleTimeout: 60_000,
|
||||
gracefulEnd: true,
|
||||
enableKeepAlive: true,
|
||||
host: 'db.internal',
|
||||
port: 3307,
|
||||
user: 'puter',
|
||||
password: 'hunter2',
|
||||
database: 'puterdb',
|
||||
multipleStatements: true,
|
||||
});
|
||||
});
|
||||
|
||||
// mysql2 skips its idle reaper unless maxIdle < connectionLimit, and an
|
||||
// unreaped connection is closed by the server under the pool's feet.
|
||||
it('keeps the idle reaper on when a pool sets its own connection limit', async () => {
|
||||
await startClient({
|
||||
replica: { host: 'replica.internal', connectionLimit: 10 },
|
||||
});
|
||||
|
||||
expect(createdPools[1].poolConfig).toMatchObject({
|
||||
connectionLimit: 10,
|
||||
maxIdle: 9,
|
||||
idleTimeout: 60_000,
|
||||
});
|
||||
});
|
||||
|
||||
// mysql2 reads 0 as unlimited; a maxIdle of -1 would crash its reaper.
|
||||
it('leaves maxIdle to mysql2 for an unlimited pool', async () => {
|
||||
await startClient({
|
||||
replica: { host: 'replica.internal', connectionLimit: 0 },
|
||||
});
|
||||
|
||||
expect(createdPools[1].poolConfig).toMatchObject({
|
||||
connectionLimit: 0,
|
||||
});
|
||||
expect(createdPools[1].poolConfig).not.toHaveProperty('maxIdle');
|
||||
});
|
||||
|
||||
it('falls back to loopback defaults when the endpoint is unspecified', async () => {
|
||||
await startClient();
|
||||
|
||||
expect(createdPools[0].poolConfig).toMatchObject({
|
||||
host: '127.0.0.1',
|
||||
port: 3306,
|
||||
user: 'root',
|
||||
password: '',
|
||||
database: 'puter',
|
||||
});
|
||||
});
|
||||
|
||||
it('arms a server-side statement timeout on every new connection', async () => {
|
||||
await startClient({ selectTimeoutMs: 12_000 });
|
||||
|
||||
const pool = createdPools[0];
|
||||
expect(pool.connectionHandlers).toHaveLength(1);
|
||||
|
||||
const conn = { query: vi.fn() };
|
||||
pool.connectionHandlers[0](conn);
|
||||
expect(conn.query).toHaveBeenCalledWith(
|
||||
'SET SESSION max_execution_time = 12000',
|
||||
);
|
||||
});
|
||||
|
||||
it('omits the statement timeout when configured to 0', async () => {
|
||||
await startClient({ selectTimeoutMs: 0 });
|
||||
expect(createdPools[0].connectionHandlers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('shares one pool between reads and writes without a replica', async () => {
|
||||
await startClient();
|
||||
expect(createPoolMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('creates a second pool when a read-replica is configured', async () => {
|
||||
await startClient({
|
||||
replica: { host: 'replica.internal', port: 3306 },
|
||||
});
|
||||
|
||||
expect(createPoolMock).toHaveBeenCalledTimes(2);
|
||||
expect(createdPools[1].poolConfig).toMatchObject({
|
||||
host: 'replica.internal',
|
||||
multipleStatements: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — query interface', () => {
|
||||
it('sends the query and its parameters through to the driver', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => rowSets([{ id: 1, username: 'ada' }]);
|
||||
|
||||
await expect(
|
||||
client.read('SELECT * FROM `user` WHERE `id` = ?', [1]),
|
||||
).resolves.toEqual([{ id: 1, username: 'ada' }]);
|
||||
|
||||
expect(pool.calls).toEqual([
|
||||
{
|
||||
sql: 'SELECT * FROM `user` WHERE `id` = ?; SELECT 1',
|
||||
values: [1],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns an empty array when the driver yields no row-set', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () => [[], undefined];
|
||||
|
||||
await expect(client.read('SELECT 1')).resolves.toEqual([]);
|
||||
await expect(client.pread('SELECT 1')).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('reads from the replica pool and primary-reads from the primary', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const [primary, replica] = createdPools;
|
||||
primary.respond = () => rowSets([{ from: 'primary' }]);
|
||||
replica.respond = () => rowSets([{ from: 'replica' }]);
|
||||
|
||||
await expect(client.read('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'replica' },
|
||||
]);
|
||||
await expect(client.pread('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'primary' },
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ method: 'pread', poolIndex: 0, maxTimeInQueue: 30 },
|
||||
{ method: 'read', poolIndex: 1, maxTimeInQueue: 10 },
|
||||
] as const)(
|
||||
'$method: flushes lone reads at once until 8 are in flight, then waits $maxTimeInQueue ms and coalesces',
|
||||
async ({ method, poolIndex, maxTimeInQueue }) => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const pool = createdPools[poolIndex];
|
||||
const held: Array<() => void> = [];
|
||||
pool.respond = (sql) =>
|
||||
new Promise((resolve) => {
|
||||
const sets = sql
|
||||
.split(';')
|
||||
.slice(0, -1)
|
||||
.map((stmt) => [{ stmt }]);
|
||||
held.push(() => resolve(rowSets(...sets)));
|
||||
});
|
||||
vi.useFakeTimers({
|
||||
toFake: [
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
try {
|
||||
const reads: Promise<unknown>[] = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
reads.push(client[method](`SELECT ${i}`));
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
}
|
||||
expect(pool.calls).toHaveLength(8);
|
||||
|
||||
reads.push(client[method]('SELECT 8'));
|
||||
await vi.advanceTimersByTimeAsync(maxTimeInQueue - 1);
|
||||
reads.push(client[method]('SELECT 9'));
|
||||
expect(pool.calls).toHaveLength(8);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(pool.calls.map((c) => c.sql).at(-1)).toBe(
|
||||
'SELECT 8;SELECT 9; SELECT 1',
|
||||
);
|
||||
|
||||
for (const release of held.splice(0)) release();
|
||||
const rows = await Promise.all(reads);
|
||||
expect(rows.at(-2)).toEqual([{ stmt: 'SELECT 8' }]);
|
||||
expect(rows.at(-1)).toEqual([{ stmt: 'SELECT 9' }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ setup: 'single node', replica: undefined, waits: true },
|
||||
{ setup: 'replica', replica: { host: 'replica' }, waits: false },
|
||||
])(
|
||||
'$setup: 8 primary reads in flight hold back a lone read: $waits',
|
||||
async ({ replica, waits }) => {
|
||||
const client = await startClient(replica ? { replica } : {});
|
||||
const held: Array<() => void> = [];
|
||||
for (const pool of createdPools) {
|
||||
pool.respond = (sql) =>
|
||||
new Promise((resolve) => {
|
||||
const sets = sql
|
||||
.split(';')
|
||||
.slice(0, -1)
|
||||
.map((stmt) => [{ stmt }]);
|
||||
held.push(() => resolve(rowSets(...sets)));
|
||||
});
|
||||
}
|
||||
const sent = () => createdPools.flatMap((pool) => pool.calls);
|
||||
vi.useFakeTimers({
|
||||
toFake: [
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
try {
|
||||
const reads: Promise<unknown>[] = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
reads.push(client.pread(`SELECT ${i}`));
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
}
|
||||
reads.push(client.read('SELECT 8'));
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toHaveLength(waits ? 8 : 9);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
expect(sent()).toHaveLength(9);
|
||||
|
||||
for (const release of held.splice(0)) release();
|
||||
await Promise.all(reads);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('sends a write on its own, outside a transaction', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => writeHeader({ affectedRows: 1 });
|
||||
|
||||
await client.write('UPDATE `user` SET `username` = ? WHERE `id` = ?', [
|
||||
'ada',
|
||||
1,
|
||||
]);
|
||||
|
||||
expect(pool.calls).toEqual([
|
||||
{
|
||||
sql: 'UPDATE `user` SET `username` = ? WHERE `id` = ?',
|
||||
values: ['ada', 1],
|
||||
},
|
||||
]);
|
||||
expect(pool.connections[0].beginTransaction).not.toHaveBeenCalled();
|
||||
expect(pool.connections[0].commit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('maps the mysql result header onto the shared write result', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () =>
|
||||
writeHeader({ insertId: 42, affectedRows: 2 });
|
||||
|
||||
await expect(
|
||||
client.write('UPDATE `user` SET `username` = ? WHERE `id` = ?', [
|
||||
'ada',
|
||||
1,
|
||||
]),
|
||||
).resolves.toEqual({
|
||||
insertId: 42,
|
||||
affectedRows: 2,
|
||||
anyRowsAffected: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('passes the warning count through, so callers can tell a clean write', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () =>
|
||||
writeHeader({ insertId: 9, affectedRows: 1, warningStatus: 1 });
|
||||
|
||||
await expect(
|
||||
client.write('INSERT IGNORE INTO `user` (`username`) VALUES (?)', [
|
||||
'ada',
|
||||
]),
|
||||
).resolves.toEqual({
|
||||
insertId: 9,
|
||||
affectedRows: 1,
|
||||
anyRowsAffected: true,
|
||||
warningStatus: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('reports no rows affected when the header omits the counters', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () => writeHeader({});
|
||||
|
||||
await expect(client.write('DELETE FROM `user`')).resolves.toEqual({
|
||||
insertId: 0,
|
||||
affectedRows: 0,
|
||||
anyRowsAffected: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('builds an INSERT with quoted identifiers and positional params', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => writeHeader({ insertId: 7, affectedRows: 1 });
|
||||
|
||||
await expect(
|
||||
client.insert('user', { username: 'ada', email: null }),
|
||||
).resolves.toMatchObject({ insertId: 7 });
|
||||
|
||||
expect(pool.calls[0]).toEqual({
|
||||
sql: 'INSERT INTO `user` (`username`, `email`) VALUES (?, ?)',
|
||||
values: ['ada', null],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — batchWrite transactions', () => {
|
||||
it('runs every statement on one connection inside a transaction', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
|
||||
await client.batchWrite([
|
||||
{ statement: 'UPDATE `user` SET `x` = ?', values: [1] },
|
||||
{
|
||||
statement: 'DELETE FROM `sessions` WHERE `uuid` = ?',
|
||||
values: ['s'],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(pool.connections).toHaveLength(1);
|
||||
const conn = pool.connections[0];
|
||||
expect(conn.beginTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(conn.execute.mock.calls).toEqual([
|
||||
['UPDATE `user` SET `x` = ?', [1]],
|
||||
['DELETE FROM `sessions` WHERE `uuid` = ?', ['s']],
|
||||
]);
|
||||
expect(conn.commit).toHaveBeenCalledTimes(1);
|
||||
expect(conn.rollback).not.toHaveBeenCalled();
|
||||
expect(conn.release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('rolls back and releases the connection when a statement fails', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = (sql) => {
|
||||
if (sql.startsWith('DELETE')) throw new Error('constraint blew up');
|
||||
return [[], undefined];
|
||||
};
|
||||
|
||||
await expect(
|
||||
client.batchWrite([
|
||||
{ statement: 'UPDATE `user` SET `x` = ?', values: [1] },
|
||||
{ statement: 'DELETE FROM `user`', values: [] },
|
||||
]),
|
||||
).rejects.toThrow('constraint blew up');
|
||||
|
||||
const conn = pool.connections[0];
|
||||
expect(conn.commit).not.toHaveBeenCalled();
|
||||
expect(conn.rollback).toHaveBeenCalledTimes(1);
|
||||
expect(conn.release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('still rethrows the original failure when the rollback also fails', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => {
|
||||
throw new Error('statement blew up');
|
||||
};
|
||||
|
||||
const original = client.batchWrite([
|
||||
{ statement: 'UPDATE `user` SET `x` = ?', values: [1] },
|
||||
]);
|
||||
// The rollback is attached after the connection is handed out, so
|
||||
// patch it once the adapter has acquired one.
|
||||
await Promise.resolve();
|
||||
pool.connections[0]?.rollback.mockRejectedValue(
|
||||
new Error('rollback blew up'),
|
||||
);
|
||||
|
||||
await expect(original).rejects.toThrow('statement blew up');
|
||||
expect(pool.connections[0].release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('never acquires a connection for an empty batch', async () => {
|
||||
const client = await startClient();
|
||||
await client.batchWrite([]);
|
||||
expect(createdPools[0].connections).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — tryHardRead', () => {
|
||||
it('issues exactly one query in single-node configuration', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => rowSets([{ id: 1 }]);
|
||||
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ id: 1 },
|
||||
]);
|
||||
expect(pool.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('prefers replica rows when the replica has them', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const [primary, replica] = createdPools;
|
||||
primary.respond = () => rowSets([{ from: 'primary' }]);
|
||||
replica.respond = () => rowSets([{ from: 'replica' }]);
|
||||
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'replica' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the primary when the replica is behind', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const [primary, replica] = createdPools;
|
||||
primary.respond = () => rowSets([{ from: 'primary' }]);
|
||||
replica.respond = () => rowSets([]);
|
||||
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'primary' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the primary when the replica query throws', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const [primary, replica] = createdPools;
|
||||
primary.respond = () => rowSets([{ from: 'primary' }]);
|
||||
replica.respond = () => {
|
||||
throw new Error('replica down');
|
||||
};
|
||||
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'primary' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('reads the primary once when the replica fails in a way read() fails over on', async () => {
|
||||
const replica = {
|
||||
execute: vi.fn().mockRejectedValue(codedError('dbBatchFailed')),
|
||||
};
|
||||
const primary = {
|
||||
execute: vi.fn().mockResolvedValue([[{ from: 'primary' }]]),
|
||||
};
|
||||
const client = makeClient({ replica, primary });
|
||||
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'primary' },
|
||||
]);
|
||||
expect(primary.execute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — migrations', () => {
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'puter-mysql-mig-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('applies mysql migrations in numeric order, one statement at a time', async () => {
|
||||
writeFileSync(
|
||||
join(dir, 'mysql_mig_2.sql'),
|
||||
'CREATE TABLE b (id INT);\nCREATE TABLE c (id INT);\n',
|
||||
);
|
||||
writeFileSync(
|
||||
join(dir, 'mysql_mig_10.sql'),
|
||||
'CREATE TABLE d (id INT);',
|
||||
);
|
||||
// Other engines' files and non-SQL noise must be ignored.
|
||||
writeFileSync(join(dir, 'postgres_mig_1.sql'), 'CREATE TABLE nope ();');
|
||||
writeFileSync(join(dir, 'mysql_notes.txt'), 'not sql');
|
||||
|
||||
await startClient({ migrationPaths: [dir] });
|
||||
|
||||
const pool = createdPools[0];
|
||||
expect(pool.calls.map((c) => c.sql)).toEqual([
|
||||
'CREATE TABLE b (id INT)',
|
||||
'CREATE TABLE c (id INT)',
|
||||
'CREATE TABLE d (id INT)',
|
||||
]);
|
||||
expect(pool.connections[0].release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reports which file and statement index failed', async () => {
|
||||
writeFileSync(
|
||||
join(dir, 'mysql_mig_1.sql'),
|
||||
'CREATE TABLE ok (id INT);\nCREATE TABLE bad (id INT);',
|
||||
);
|
||||
|
||||
const client = new MySQLDatabaseClient(
|
||||
mysqlConfig({ migrationPaths: [dir] }),
|
||||
);
|
||||
const cause = new Error('ER_PARSE_ERROR');
|
||||
// The pool only exists once onServerStart runs, so arm the failure
|
||||
// through the factory.
|
||||
createPoolMock.mockImplementationOnce((poolConfig) => {
|
||||
const pool = createPoolMock.getMockImplementation()!(poolConfig);
|
||||
pool.respond = (sql: string) => {
|
||||
if (sql.includes('bad')) throw cause;
|
||||
return [[], undefined];
|
||||
};
|
||||
return pool;
|
||||
});
|
||||
|
||||
await expect(client.onServerStart()).rejects.toThrow(
|
||||
'[mysql] failed to apply mysql_mig_1.sql at statement 1',
|
||||
);
|
||||
});
|
||||
|
||||
it('fails loudly when a configured migration path is unreadable', async () => {
|
||||
await expect(
|
||||
startClient({ migrationPaths: [join(dir, 'does-not-exist')] }),
|
||||
).rejects.toThrow('[mysql] migration path is unreadable');
|
||||
});
|
||||
|
||||
it('skips a configured directory that holds no mysql migrations', async () => {
|
||||
writeFileSync(join(dir, 'readme.md'), '# nothing here');
|
||||
await startClient({ migrationPaths: [dir] });
|
||||
expect(createdPools[0].calls).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — pool lifecycle', () => {
|
||||
it('closes both pools on shutdown when a replica is configured', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
await client.onServerShutdown();
|
||||
|
||||
expect(createdPools[0].ended).toBe(true);
|
||||
expect(createdPools[1].ended).toBe(true);
|
||||
});
|
||||
|
||||
it('leaves the pools open through prepare-shutdown; only onServerShutdown closes them', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const client = await startClient();
|
||||
await client.onServerPrepareShutdown();
|
||||
expect(createdPools[0].ended).toBe(false);
|
||||
|
||||
// No timer closes the pools on its own — the layers above are
|
||||
// still draining through them for as long as the process drains.
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(createdPools[0].ended).toBe(false);
|
||||
|
||||
await client.onServerShutdown();
|
||||
expect(createdPools[0].ended).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('surfaces a driver error raised while closing the pool', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].endError = new Error('pool refused to close');
|
||||
|
||||
await expect(client.onServerShutdown()).rejects.toThrow(
|
||||
'pool refused to close',
|
||||
);
|
||||
});
|
||||
|
||||
it('surfaces a driver error thrown synchronously by end()', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].endThrows = new Error('end exploded');
|
||||
|
||||
await expect(client.onServerShutdown()).rejects.toThrow('end exploded');
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
/*
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
@@ -17,27 +17,32 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { metrics } from '@opentelemetry/api';
|
||||
import { createPool, ExecuteValues, type Pool } from 'mysql2';
|
||||
import { Span } from '../../util/span.js';
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { isAbsolute, resolve as resolvePath } from 'path';
|
||||
import { createPool, type Pool } from 'mysql2';
|
||||
import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
|
||||
import { SQLBatcher } from './SQLBatcher.js';
|
||||
import { isRetriableError } from './retriableErrors.js';
|
||||
import { splitMysqlStatements } from './splitMysqlStatements.js';
|
||||
import type { IConfig } from '../../types';
|
||||
|
||||
const DEFAULT_SELECT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_CONNECTION_LIMIT = 30;
|
||||
const IDLE_TIMEOUT_MS = 60_000;
|
||||
const RETRIABLE_ERROR_CODES = new Set([
|
||||
'PROTOCOL_CONNECTION_LOST',
|
||||
'PROTOCOL_SEQUENCE_TIMEOUT',
|
||||
'PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR',
|
||||
'ECONNRESET',
|
||||
'ETIMEDOUT',
|
||||
'EPIPE',
|
||||
'ECONNREFUSED',
|
||||
'EHOSTUNREACH',
|
||||
'ENETUNREACH',
|
||||
'EAI_AGAIN',
|
||||
]);
|
||||
|
||||
const replicaFailoverCounter = metrics
|
||||
.getMeter('puter-backend')
|
||||
.createCounter('db.read.replica_failover', {
|
||||
description:
|
||||
'Reads that failed on the replica batcher and were retried on the primary',
|
||||
});
|
||||
|
||||
export { compareMigrationFilenames } from './migrationFilenames.js';
|
||||
const RETRIABLE_ERROR_MESSAGES = [
|
||||
'Connection lost',
|
||||
'read ECONNRESET',
|
||||
'ETIMEDOUT',
|
||||
];
|
||||
|
||||
type PoolConfig = Parameters<typeof createPool>[0];
|
||||
|
||||
@@ -53,10 +58,9 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
private replicaPool!: Pool;
|
||||
private db!: SQLBatcher;
|
||||
private dbReplica!: SQLBatcher;
|
||||
/** Primary pool, SELECT-only: same rows as `db`, coalesced like the replica. */
|
||||
private dbPrimaryRead!: SQLBatcher;
|
||||
private configuration = Configuration.SINGLE;
|
||||
private shutdownStarted = false;
|
||||
private shutdownTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(config: IConfig) {
|
||||
super(config);
|
||||
@@ -78,13 +82,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
});
|
||||
console.log('[mysql] connected to primary');
|
||||
|
||||
// Read batchers on one pool share the cap on immediate flushes.
|
||||
const primaryReadFlushes = { count: 0 };
|
||||
this.db = this.createPrimaryBatcher(this.primaryPool);
|
||||
this.dbPrimaryRead = this.createPrimaryReadBatcher(
|
||||
this.primaryPool,
|
||||
primaryReadFlushes,
|
||||
);
|
||||
this.db = new SQLBatcher(this.primaryPool, 30, 5);
|
||||
|
||||
if (dbConf.replica) {
|
||||
this.replicaPool = this.createPool(dbConf.replica);
|
||||
@@ -95,30 +93,38 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
this.configuration = Configuration.SINGLE;
|
||||
}
|
||||
|
||||
this.dbReplica = this.createReplicaBatcher(
|
||||
this.replicaPool,
|
||||
this.configuration === Configuration.SINGLE
|
||||
? primaryReadFlushes
|
||||
: { count: 0 },
|
||||
);
|
||||
this.dbReplica = new SQLBatcher(this.replicaPool, 10, 5);
|
||||
|
||||
await this.applyMigrationPaths(
|
||||
() => this.primaryPool.promise().getConnection(),
|
||||
splitMysqlStatements,
|
||||
{ transactional: false },
|
||||
);
|
||||
await this.runMigrations();
|
||||
}
|
||||
|
||||
override async onServerPrepareShutdown(): Promise<void> {
|
||||
if (this.shutdownStarted) return;
|
||||
this.shutdownStarted = true;
|
||||
|
||||
// Pools stay open; onServerShutdown closes them after the layers
|
||||
// above have drained.
|
||||
console.log('[mysql] entering drain mode');
|
||||
// Allow in-flight queries to drain before closing pools
|
||||
const drainMs = 60_000;
|
||||
console.log(
|
||||
`[mysql] draining in-flight queries (${drainMs}ms) before closing pools`,
|
||||
);
|
||||
|
||||
this.shutdownTimer = setTimeout(() => {
|
||||
this.shutdownTimer = null;
|
||||
this.closeCurrentPools('drain').catch((e) =>
|
||||
console.error('[mysql] error closing pools after drain', e),
|
||||
);
|
||||
}, drainMs);
|
||||
|
||||
if (typeof this.shutdownTimer.unref === 'function') {
|
||||
this.shutdownTimer.unref();
|
||||
}
|
||||
}
|
||||
|
||||
override async onServerShutdown(): Promise<void> {
|
||||
if (this.shutdownTimer) {
|
||||
clearTimeout(this.shutdownTimer);
|
||||
this.shutdownTimer = null;
|
||||
}
|
||||
await this.closeCurrentPools('shutdown');
|
||||
}
|
||||
|
||||
@@ -126,46 +132,24 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
// Query interface
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
// The db.* spans measure the logical query, including time queued in
|
||||
// the SQLBatcher — the mysql2 auto-instrumentation only sees the
|
||||
// coalesced multi-statement flush, so per-query latency lives here.
|
||||
@Span('db.read', (query: string) => ({ 'db.statement': query }))
|
||||
override async read(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
let result;
|
||||
try {
|
||||
result = await this.dbReplica.execute(query, params);
|
||||
} catch (error) {
|
||||
// Replica-side degradation (batcher load-shed or a transient
|
||||
// connection failure) shouldn't fail reads while the primary is
|
||||
// healthy. Deterministic errors (bad SQL) are rethrown — they
|
||||
// would fail identically on the primary.
|
||||
if (
|
||||
this.configuration !== Configuration.REPLICA ||
|
||||
!MySQLDatabaseClient.isFailoverWorthy(error)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
replicaFailoverCounter.add(1);
|
||||
result = await this.dbPrimaryRead.execute(query, params);
|
||||
}
|
||||
const result = await this.dbReplica.execute(query, params);
|
||||
if (!result) return [];
|
||||
return (result[0] as Record<string, unknown>[]) ?? [];
|
||||
}
|
||||
|
||||
@Span('db.pread', (query: string) => ({ 'db.statement': query }))
|
||||
override async pread(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const result = await this.dbPrimaryRead.execute(query, params);
|
||||
const result = await this.db.execute(query, params);
|
||||
if (!result) return [];
|
||||
return (result[0] as Record<string, unknown>[]) ?? [];
|
||||
}
|
||||
|
||||
@Span('db.write', (query: string) => ({ 'db.statement': query }))
|
||||
override async write(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
@@ -174,20 +158,15 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
const header = result[0] as {
|
||||
insertId?: number;
|
||||
affectedRows?: number;
|
||||
warningStatus?: number;
|
||||
};
|
||||
const affectedRows = header.affectedRows ?? 0;
|
||||
return {
|
||||
insertId: header.insertId ?? 0,
|
||||
affectedRows,
|
||||
anyRowsAffected: affectedRows > 0,
|
||||
warningStatus: header.warningStatus,
|
||||
};
|
||||
}
|
||||
|
||||
@Span('db.batchWrite', (entries: unknown[]) => ({
|
||||
'db.batch_size': entries.length,
|
||||
}))
|
||||
override async batchWrite(
|
||||
entries: { statement: string; values: unknown[] }[],
|
||||
): Promise<void> {
|
||||
@@ -201,7 +180,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
await conn.beginTransaction();
|
||||
try {
|
||||
for (const { statement, values } of entries) {
|
||||
await conn.execute(statement, values as ExecuteValues);
|
||||
await conn.execute(statement, values);
|
||||
}
|
||||
await conn.commit();
|
||||
} catch (err) {
|
||||
@@ -213,19 +192,100 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
}
|
||||
}
|
||||
|
||||
protected override hasReadReplica(): boolean {
|
||||
return this.configuration === Configuration.REPLICA;
|
||||
override async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (this.configuration === Configuration.SINGLE) {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
// Run both reads in parallel — prefer replica when it returns rows,
|
||||
// otherwise fall back to primary to handle replication lag.
|
||||
const primaryPromise = this.db.execute(query, params);
|
||||
try {
|
||||
const replicaResult = await this.dbReplica.execute(query, params);
|
||||
if (
|
||||
Array.isArray(replicaResult?.[0]) &&
|
||||
(replicaResult[0] as unknown[]).length > 0
|
||||
) {
|
||||
primaryPromise.catch(() => {}); // suppress unhandled rejection
|
||||
return replicaResult[0] as Record<string, unknown>[];
|
||||
}
|
||||
} catch {
|
||||
// fall through to primary
|
||||
}
|
||||
|
||||
const primaryResult = await primaryPromise;
|
||||
return (primaryResult?.[0] as Record<string, unknown>[]) ?? [];
|
||||
}
|
||||
|
||||
// No failover: `tryHardRead` falls back to the primary on an empty result
|
||||
// or error (raced alongside by default, after the replica with `race: false`).
|
||||
protected override async replicaRead(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const result = await this.dbReplica.execute(query, params);
|
||||
if (!result) return [];
|
||||
return (result[0] as Record<string, unknown>[]) ?? [];
|
||||
// ------------------------------------------------------------------
|
||||
// Migrations
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Apply `.sql` files from each configured migration directory in order.
|
||||
* Files within a directory are sorted lexically. Files MUST be
|
||||
* idempotent — there is no per-file applied-state tracking. Failures
|
||||
* abort startup so operators see schema problems loud.
|
||||
*/
|
||||
private async runMigrations(): Promise<void> {
|
||||
const paths = this.config.database?.migrationPaths;
|
||||
if (!paths || paths.length === 0) return;
|
||||
|
||||
const conn = await this.primaryPool.promise().getConnection();
|
||||
try {
|
||||
for (const rawPath of paths) {
|
||||
const dir = isAbsolute(rawPath)
|
||||
? rawPath
|
||||
: resolvePath(process.cwd(), rawPath);
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = readdirSync(dir)
|
||||
.filter(
|
||||
(f) => f.endsWith('.sql') && f.startsWith('mysql'),
|
||||
)
|
||||
.sort();
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`[mysql] migration path is unreadable: ${dir}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log(`[mysql] no migrations in ${dir}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[mysql] running migrations from ${dir}: ${files.length} file(s)`,
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = resolvePath(dir, file);
|
||||
const contents = readFileSync(filePath, 'utf8');
|
||||
const statements = splitMysqlStatements(contents);
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
try {
|
||||
await conn.query(statements[i]);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`[mysql] failed to apply ${file} at statement ${i}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`[mysql] applied ${file} (${statements.length} statements)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
@@ -233,97 +293,112 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private createPool(poolConfig: PoolConfig): Pool {
|
||||
const connectionLimit =
|
||||
poolConfig.connectionLimit ?? DEFAULT_CONNECTION_LIMIT;
|
||||
const pool = createPool({
|
||||
return createPool({
|
||||
maxPreparedStatements: 900,
|
||||
connectionLimit,
|
||||
// mysql2 only reaps idle connections when maxIdle < connectionLimit.
|
||||
// Unreaped, they sit until the server's wait_timeout closes them,
|
||||
// and the next statement sent on one fails. 0 means unlimited,
|
||||
// and a negative maxIdle would crash the reaper.
|
||||
...(connectionLimit > 1 && { maxIdle: connectionLimit - 1 }),
|
||||
idleTimeout: IDLE_TIMEOUT_MS,
|
||||
// Reaped connections say goodbye instead of dropping the socket.
|
||||
gracefulEnd: true,
|
||||
enableKeepAlive: true,
|
||||
connectionLimit: 30,
|
||||
...poolConfig,
|
||||
multipleStatements: true,
|
||||
} as PoolConfig);
|
||||
}
|
||||
|
||||
// Server-side kill switch for runaway reads: MySQL applies
|
||||
// max_execution_time to SELECT statements only, so this is
|
||||
// write-safe. Without it, a stalled database turns reads into
|
||||
// indefinite hangs that no client-side timeout ever converts
|
||||
// into a failure. 0 disables.
|
||||
const selectTimeoutMs = Math.floor(
|
||||
Number(
|
||||
this.config.database?.selectTimeoutMs ??
|
||||
DEFAULT_SELECT_TIMEOUT_MS,
|
||||
),
|
||||
);
|
||||
if (selectTimeoutMs > 0) {
|
||||
pool.on('connection', (conn) => {
|
||||
conn.query(
|
||||
`SET SESSION max_execution_time = ${selectTimeoutMs}`,
|
||||
);
|
||||
});
|
||||
/** Reinitialize the primary pool (e.g. after a health-check failure). */
|
||||
reinitPrimary(): void {
|
||||
if (this.shutdownStarted) return;
|
||||
|
||||
const dbConf = this.config.database!;
|
||||
const previous = this.primaryPool;
|
||||
this.primaryPool = this.createPool({
|
||||
host: dbConf.host ?? '127.0.0.1',
|
||||
port: dbConf.port ?? 3306,
|
||||
user: dbConf.user ?? 'root',
|
||||
password: dbConf.password ?? '',
|
||||
database: dbConf.database ?? 'puter',
|
||||
});
|
||||
this.db = new SQLBatcher(this.primaryPool, 30, 5);
|
||||
|
||||
if (this.configuration === Configuration.SINGLE) {
|
||||
this.replicaPool = this.primaryPool;
|
||||
this.dbReplica = new SQLBatcher(this.primaryPool, 10, 5);
|
||||
}
|
||||
|
||||
return pool;
|
||||
if (previous && previous !== this.primaryPool) {
|
||||
this.closePool(previous, 'reinit:primary').catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One write per flush. Coalesced writes share a transaction, so unrelated
|
||||
* callers hold each other's row locks until COMMIT and deadlock; a lone
|
||||
* statement needs no BEGIN/COMMIT round trips.
|
||||
*/
|
||||
private createPrimaryBatcher(pool: Pool): SQLBatcher {
|
||||
return new SQLBatcher(pool, {
|
||||
maxBatchSize: 1,
|
||||
poolLabel: 'primary',
|
||||
acquireTimeoutMs: this.config.database?.acquireTimeoutMs,
|
||||
});
|
||||
/** Reinitialize the replica pool. */
|
||||
reinitReplica(): void {
|
||||
if (this.shutdownStarted || !this.config.database?.replica) return;
|
||||
|
||||
const previous = this.replicaPool;
|
||||
this.replicaPool = this.createPool(this.config.database.replica);
|
||||
this.dbReplica = new SQLBatcher(this.replicaPool, 10, 5);
|
||||
|
||||
if (
|
||||
previous &&
|
||||
previous !== this.replicaPool &&
|
||||
previous !== this.primaryPool
|
||||
) {
|
||||
this.closePool(previous, 'reinit:replica').catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads that must see the primary, still coalesced. */
|
||||
private createPrimaryReadBatcher(
|
||||
pool: Pool,
|
||||
inFlight: { count: number },
|
||||
): SQLBatcher {
|
||||
return new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 8,
|
||||
inFlight,
|
||||
maxBatchSize: 5,
|
||||
poolLabel: 'primary',
|
||||
readOnly: true,
|
||||
acquireTimeoutMs: this.config.database?.acquireTimeoutMs,
|
||||
});
|
||||
}
|
||||
// ------------------------------------------------------------------
|
||||
// Retry helpers (for health checks or resilient reads)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private createReplicaBatcher(
|
||||
pool: Pool,
|
||||
inFlight: { count: number },
|
||||
): SQLBatcher {
|
||||
return new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 10,
|
||||
maxInFlightImmediate: 8,
|
||||
inFlight,
|
||||
maxBatchSize: 5,
|
||||
poolLabel: 'replica',
|
||||
readOnly: true,
|
||||
acquireTimeoutMs: this.config.database?.acquireTimeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Replica failures worth retrying on the primary: batcher load-shed or
|
||||
* transient connection errors — never deterministic SQL errors.
|
||||
*/
|
||||
private static isFailoverWorthy(error: unknown): boolean {
|
||||
static isRetriableError(error: unknown): boolean {
|
||||
const code = (error as { code?: string })?.code;
|
||||
return code === 'dbBatchFailed' || isRetriableError(error);
|
||||
if (code && RETRIABLE_ERROR_CODES.has(code)) return true;
|
||||
|
||||
const msg = String((error as Error)?.message ?? '');
|
||||
return RETRIABLE_ERROR_MESSAGES.some((m) => msg.includes(m));
|
||||
}
|
||||
|
||||
async readWithRetry(
|
||||
label: string,
|
||||
operation: () => Promise<unknown[]>,
|
||||
opts?: {
|
||||
maxAttempts?: number;
|
||||
baseBackoffMs?: number;
|
||||
maxBackoffMs?: number;
|
||||
jitterRatio?: number;
|
||||
},
|
||||
): Promise<unknown[]> {
|
||||
const maxAttempts = opts?.maxAttempts ?? 3;
|
||||
const baseBackoffMs = opts?.baseBackoffMs ?? 100;
|
||||
const maxBackoffMs = opts?.maxBackoffMs ?? 500;
|
||||
const jitterRatio = opts?.jitterRatio ?? 0.2;
|
||||
|
||||
let attempt = 1;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
if (this.shutdownStarted) throw error;
|
||||
if (
|
||||
attempt >= maxAttempts ||
|
||||
!MySQLDatabaseClient.isRetriableError(error)
|
||||
)
|
||||
throw error;
|
||||
|
||||
const raw = baseBackoffMs * 2 ** (attempt - 1);
|
||||
const capped = Math.min(maxBackoffMs, raw);
|
||||
const window = Math.round(capped * jitterRatio);
|
||||
const jitter =
|
||||
window === 0
|
||||
? 0
|
||||
: Math.floor(Math.random() * (window * 2 + 1)) - window;
|
||||
const delay = Math.max(0, capped + jitter);
|
||||
|
||||
console.warn(
|
||||
`[${label}] transient mysql error (${(error as { code?: string })?.code ?? 'unknown'}); retry ${attempt + 1}/${maxAttempts} in ${delay}ms`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
attempt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
@@ -1,328 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import type { IConfig } from '../../types';
|
||||
import type { PuterServer } from '../../server';
|
||||
import { generateDefaultFsentries } from '../../util/userProvisioning.js';
|
||||
import {
|
||||
createPgMockPostgresDatabaseClient,
|
||||
POSTGRES_TEST_MIGRATIONS_PATH,
|
||||
setupTestServer,
|
||||
} from '../../testUtil.js';
|
||||
import { PostgresDatabaseClient } from './PostgresDatabaseClient.js';
|
||||
|
||||
const postgresUrl = process.env.PUTER_TEST_POSTGRES_URL;
|
||||
const postgresMigrationsPath = POSTGRES_TEST_MIGRATIONS_PATH;
|
||||
const postgresTestSchemaPattern = /^puter_test_[a-f0-9]{32}$/u;
|
||||
const postgresIntegrationTimeoutMs = 180_000;
|
||||
|
||||
let postgresTestSchema: string | undefined;
|
||||
let postgresTestUrl: string | undefined;
|
||||
|
||||
const postgresConfig = (overrides: Partial<IConfig> = {}): IConfig => {
|
||||
if (postgresUrl && !postgresTestUrl) {
|
||||
throw new Error('Postgres test schema was not initialized');
|
||||
}
|
||||
|
||||
const { database: databaseOverrides, ...rootOverrides } = overrides;
|
||||
const database = postgresUrl
|
||||
? {
|
||||
engine: 'postgres' as const,
|
||||
inMemory: false,
|
||||
connectionString: postgresTestUrl,
|
||||
migrationPaths: [postgresMigrationsPath],
|
||||
}
|
||||
: {
|
||||
engine: 'postgres' as const,
|
||||
inMemory: true,
|
||||
migrationPaths: [postgresMigrationsPath],
|
||||
};
|
||||
|
||||
return {
|
||||
port: 0,
|
||||
extensions: [],
|
||||
database: { ...database, ...(databaseOverrides ?? {}) },
|
||||
...rootOverrides,
|
||||
};
|
||||
};
|
||||
|
||||
const quoteTestSchemaIdentifier = (schema: string): string => {
|
||||
if (!postgresTestSchemaPattern.test(schema)) {
|
||||
throw new Error(`Unsafe Postgres test schema name: ${schema}`);
|
||||
}
|
||||
return `"${schema}"`;
|
||||
};
|
||||
|
||||
const postgresConnectionStringForSchema = (
|
||||
connectionString: string,
|
||||
schema: string,
|
||||
): string => {
|
||||
if (!postgresTestSchemaPattern.test(schema)) {
|
||||
throw new Error(`Unsafe Postgres test schema name: ${schema}`);
|
||||
}
|
||||
|
||||
const url = new URL(connectionString);
|
||||
url.searchParams.set('options', `-c search_path=${schema}`);
|
||||
return url.toString();
|
||||
};
|
||||
|
||||
const createPostgresTestSchema = async (): Promise<void> => {
|
||||
if (!postgresUrl) return;
|
||||
|
||||
const schema = `puter_test_${uuidv4().replaceAll('-', '')}`;
|
||||
const pool = new Pool({ connectionString: postgresUrl });
|
||||
try {
|
||||
await pool.query(`CREATE SCHEMA ${quoteTestSchemaIdentifier(schema)}`);
|
||||
postgresTestSchema = schema;
|
||||
postgresTestUrl = postgresConnectionStringForSchema(
|
||||
postgresUrl,
|
||||
schema,
|
||||
);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
};
|
||||
|
||||
const dropPostgresTestSchema = async (): Promise<void> => {
|
||||
if (!postgresUrl || !postgresTestSchema) return;
|
||||
|
||||
const schema = postgresTestSchema;
|
||||
postgresTestSchema = undefined;
|
||||
postgresTestUrl = undefined;
|
||||
|
||||
const pool = new Pool({ connectionString: postgresUrl });
|
||||
try {
|
||||
await pool.query(
|
||||
`DROP SCHEMA IF EXISTS ${quoteTestSchemaIdentifier(schema)} CASCADE`,
|
||||
);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
};
|
||||
|
||||
describe('PostgresDatabaseClient integration', () => {
|
||||
let server: PuterServer | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
await createPostgresTestSchema();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await server?.shutdown();
|
||||
} finally {
|
||||
server = undefined;
|
||||
await dropPostgresTestSchema();
|
||||
}
|
||||
});
|
||||
|
||||
it(
|
||||
'applies the native migrations idempotently to an empty database',
|
||||
async () => {
|
||||
const config = postgresConfig();
|
||||
const pgMockClient = postgresUrl
|
||||
? undefined
|
||||
: await createPgMockPostgresDatabaseClient(config);
|
||||
try {
|
||||
const firstClient =
|
||||
pgMockClient?.client ?? new PostgresDatabaseClient(config);
|
||||
try {
|
||||
await firstClient.onServerStart();
|
||||
} finally {
|
||||
await firstClient.onServerShutdown();
|
||||
}
|
||||
|
||||
const secondClient =
|
||||
pgMockClient?.createClient() ??
|
||||
new PostgresDatabaseClient(config);
|
||||
try {
|
||||
await secondClient.onServerStart();
|
||||
const [systemUser] = await secondClient.read(
|
||||
'SELECT `id`, `username` FROM `user` WHERE `username` = ?',
|
||||
['system'],
|
||||
);
|
||||
const [devCenter] = await secondClient.read(
|
||||
'SELECT `name`, `index_url` FROM `apps` WHERE `name` = ?',
|
||||
['dev-center'],
|
||||
);
|
||||
|
||||
expect(systemUser).toMatchObject({
|
||||
id: 1,
|
||||
username: 'system',
|
||||
});
|
||||
expect(devCenter).toMatchObject({
|
||||
name: 'dev-center',
|
||||
index_url:
|
||||
'https://builtins.namespaces.puter.com/dev-center',
|
||||
});
|
||||
} finally {
|
||||
await secondClient.onServerShutdown();
|
||||
}
|
||||
} finally {
|
||||
pgMockClient?.destroy();
|
||||
}
|
||||
},
|
||||
postgresIntegrationTimeoutMs,
|
||||
);
|
||||
|
||||
it('starts the server and exercises user, app, fsentry, permission, and session flows', async () => {
|
||||
const userStorageAllowance = 123_456_789;
|
||||
server = await setupTestServer(
|
||||
postgresConfig({
|
||||
no_default_user: false,
|
||||
is_storage_limited: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const admin = await server.stores.user.getByUsername('admin');
|
||||
expect(admin?.username).toBe('admin');
|
||||
|
||||
const username = `pg-${uuidv4().slice(0, 8)}`;
|
||||
const createdUser = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: userStorageAllowance,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
await generateDefaultFsentries(
|
||||
server.clients.db,
|
||||
server.stores.user,
|
||||
createdUser,
|
||||
);
|
||||
const user = await server.stores.user.getById(createdUser.id);
|
||||
if (!user) throw new Error('created user was not readable');
|
||||
expect(user.username).toBe(username);
|
||||
await expect(
|
||||
server.stores.fsEntry.getUserStorageAllowance(user.id),
|
||||
).resolves.toMatchObject({
|
||||
max: userStorageAllowance,
|
||||
});
|
||||
|
||||
const otherUsername = `pg-other-${uuidv4().slice(0, 8)}`;
|
||||
const otherUser = await server.stores.user.create({
|
||||
username: otherUsername,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${otherUsername}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
|
||||
const authResult = await server.services.auth.createSessionToken(
|
||||
user,
|
||||
{
|
||||
ip: '127.0.0.1',
|
||||
user_agent: 'postgres-integration-test',
|
||||
},
|
||||
);
|
||||
const authenticated =
|
||||
await server.services.auth.authenticateFromToken(authResult.token);
|
||||
expect(authenticated?.user?.id).toBe(user?.id);
|
||||
|
||||
const devCenter = await server.stores.app.getByName('dev-center');
|
||||
if (!devCenter) throw new Error('dev-center app was not seeded');
|
||||
expect(devCenter.name).toBe('dev-center');
|
||||
|
||||
const documents = await server.stores.fsEntry.getEntryByPath(
|
||||
`/${username}/Documents`,
|
||||
);
|
||||
if (!documents) throw new Error('Documents directory was not created');
|
||||
expect(documents.isDir).toBe(true);
|
||||
const folder = await server.stores.fsEntry.createNonFileEntry({
|
||||
userId: user.id,
|
||||
parent: documents,
|
||||
name: 'postgres-folder',
|
||||
kind: 'directory',
|
||||
});
|
||||
const renamedFolder = await server.stores.fsEntry.updateEntry(
|
||||
folder.uuid,
|
||||
{
|
||||
name: 'postgres-folder-renamed',
|
||||
path: `/${username}/Documents/postgres-folder-renamed`,
|
||||
},
|
||||
);
|
||||
expect(renamedFolder.name).toBe('postgres-folder-renamed');
|
||||
|
||||
await server.stores.permission.upsertUserAppPerm(
|
||||
user.id,
|
||||
Number(devCenter.id),
|
||||
'driver:postgres-integration',
|
||||
{ ok: true },
|
||||
);
|
||||
await expect(
|
||||
server.stores.permission.hasUserAppPerm(
|
||||
user.id,
|
||||
Number(devCenter.id),
|
||||
'driver:postgres-integration',
|
||||
),
|
||||
).resolves.toBe(true);
|
||||
|
||||
await server.stores.oidc.link(
|
||||
user.id,
|
||||
'postgres-integration-test',
|
||||
'subject-1',
|
||||
null,
|
||||
);
|
||||
await expect(
|
||||
server.stores.oidc.link(
|
||||
user.id,
|
||||
'postgres-integration-test',
|
||||
'subject-1',
|
||||
null,
|
||||
),
|
||||
).resolves.toBeUndefined();
|
||||
await expect(
|
||||
server.stores.oidc.link(
|
||||
otherUser.id,
|
||||
'postgres-integration-test',
|
||||
'subject-1',
|
||||
null,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 409,
|
||||
legacyCode: 'conflict',
|
||||
});
|
||||
|
||||
const session = await server.stores.session.create(user.id, {
|
||||
meta: { source: 'postgres-integration-test' },
|
||||
});
|
||||
const activeSession = await server.stores.session.getByUuid(
|
||||
session.uuid,
|
||||
);
|
||||
expect(activeSession?.uuid).toBe(session.uuid);
|
||||
|
||||
const workerName = `worker-${uuidv4().slice(0, 8)}`;
|
||||
const workerSession = await server.stores.session.getOrCreateWorker(
|
||||
user.id,
|
||||
{ workerName },
|
||||
);
|
||||
expect(workerSession?.kind).toBe('worker');
|
||||
expect(workerSession?.meta?.worker_name).toBe(workerName);
|
||||
|
||||
await server.stores.session.removeByUuid(session.uuid);
|
||||
await expect(
|
||||
server.stores.session.getByUuid(session.uuid),
|
||||
).resolves.toBeNull();
|
||||
}, postgresIntegrationTimeoutMs);
|
||||
});
|
||||
@@ -1,420 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { FieldDef, QueryResult } from 'pg';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { IConfig } from '../../types';
|
||||
import { DatabaseClientFactory } from './index.js';
|
||||
import {
|
||||
mapPostgresWriteResult,
|
||||
PostgresDatabaseClient,
|
||||
type PostgresPool,
|
||||
type PostgresPoolClient,
|
||||
} from './PostgresDatabaseClient.js';
|
||||
|
||||
// Pass-through spy: the rewrite is pure, so its call count shows the memo.
|
||||
const { rewriteSpy } = vi.hoisted(() => ({ rewriteSpy: vi.fn() }));
|
||||
vi.mock('./preparePostgresSql.js', async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import('./preparePostgresSql.js')>();
|
||||
return {
|
||||
...actual,
|
||||
preparePostgresSql: (sql: string) => {
|
||||
rewriteSpy(sql);
|
||||
return actual.preparePostgresSql(sql);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
type QueryCall = {
|
||||
text: string;
|
||||
values?: unknown[];
|
||||
};
|
||||
|
||||
const postgresConfig = (): IConfig => ({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
database: {
|
||||
engine: 'postgres',
|
||||
migrationPaths: [],
|
||||
},
|
||||
});
|
||||
|
||||
const postgresReplicaConfig = (): IConfig => ({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
database: {
|
||||
engine: 'postgres',
|
||||
migrationPaths: [],
|
||||
replica: {},
|
||||
},
|
||||
});
|
||||
|
||||
const field = (name: string, dataTypeID: number): FieldDef => ({
|
||||
name,
|
||||
tableID: 0,
|
||||
columnID: 0,
|
||||
dataTypeID,
|
||||
dataTypeSize: -1,
|
||||
dataTypeModifier: -1,
|
||||
format: 'text',
|
||||
});
|
||||
|
||||
const int8Field = (name: string): FieldDef => field(name, 20);
|
||||
const textField = (name: string): FieldDef => field(name, 25);
|
||||
|
||||
const queryResult = (
|
||||
rows: Record<string, unknown>[] = [],
|
||||
rowCount = rows.length,
|
||||
fields: FieldDef[] = [],
|
||||
): QueryResult<Record<string, unknown>> => ({
|
||||
command: '',
|
||||
fields,
|
||||
oid: 0,
|
||||
rowCount,
|
||||
rows,
|
||||
});
|
||||
|
||||
class RecordingPoolClient implements PostgresPoolClient {
|
||||
readonly calls: QueryCall[] = [];
|
||||
released = false;
|
||||
|
||||
constructor(private readonly failOnText?: string) {}
|
||||
|
||||
async query(
|
||||
text: string,
|
||||
values?: unknown[],
|
||||
): Promise<QueryResult<Record<string, unknown>>> {
|
||||
this.calls.push({ text, values });
|
||||
if (this.failOnText && text.includes(this.failOnText)) {
|
||||
throw new Error(`forced query failure: ${text}`);
|
||||
}
|
||||
return queryResult([], text === 'ROLLBACK' ? 0 : 1);
|
||||
}
|
||||
|
||||
release(): void {
|
||||
this.released = true;
|
||||
}
|
||||
}
|
||||
|
||||
class RecordingPool implements PostgresPool {
|
||||
readonly calls: QueryCall[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly client: RecordingPoolClient =
|
||||
new RecordingPoolClient(),
|
||||
private readonly nextResult: QueryResult<Record<string, unknown>> =
|
||||
queryResult(),
|
||||
) {}
|
||||
|
||||
async query(
|
||||
text: string,
|
||||
values?: unknown[],
|
||||
): Promise<QueryResult<Record<string, unknown>>> {
|
||||
this.calls.push({ text, values });
|
||||
if (text === 'SELECT 1') return queryResult([{ ok: 1 }], 1);
|
||||
return this.nextResult;
|
||||
}
|
||||
|
||||
async connect(): Promise<PostgresPoolClient> {
|
||||
return this.client;
|
||||
}
|
||||
|
||||
async end(): Promise<void> {}
|
||||
}
|
||||
|
||||
describe('PostgresDatabaseClient', () => {
|
||||
it('is selected by the database factory', () => {
|
||||
const client = new DatabaseClientFactory(postgresConfig());
|
||||
expect(client).toBeInstanceOf(PostgresDatabaseClient);
|
||||
});
|
||||
|
||||
it('maps pg write results to the shared WriteResult shape', () => {
|
||||
expect(
|
||||
mapPostgresWriteResult(queryResult([{ id: '42' }], 1)),
|
||||
).toEqual({
|
||||
insertId: 42,
|
||||
affectedRows: 1,
|
||||
anyRowsAffected: true,
|
||||
});
|
||||
|
||||
expect(mapPostgresWriteResult(queryResult([], 0))).toEqual({
|
||||
insertId: 0,
|
||||
affectedRows: 0,
|
||||
anyRowsAffected: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('prepares write SQL at the pg boundary', async () => {
|
||||
const pool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult([{ id: '7' }], 1),
|
||||
);
|
||||
const client = new PostgresDatabaseClient(postgresConfig(), () => pool);
|
||||
await client.onServerStart();
|
||||
|
||||
const result = await client.write(
|
||||
'INSERT INTO `apps` (`name`) VALUES (?) RETURNING id',
|
||||
['editor'],
|
||||
);
|
||||
|
||||
expect(result.insertId).toBe(7);
|
||||
expect(pool.calls.at(-1)).toEqual({
|
||||
text: 'INSERT INTO "apps" ("name") VALUES ($1) RETURNING id',
|
||||
values: ['editor'],
|
||||
});
|
||||
});
|
||||
|
||||
it('rewrites a repeated query once', async () => {
|
||||
const pool = new RecordingPool();
|
||||
const client = new PostgresDatabaseClient(postgresConfig(), () => pool);
|
||||
await client.onServerStart();
|
||||
const query = 'SELECT `name` FROM `apps` WHERE `id` = ?';
|
||||
rewriteSpy.mockClear();
|
||||
|
||||
await client.read(query, [1]);
|
||||
await client.pread(query, [2]);
|
||||
await client.write(query, [3]);
|
||||
|
||||
expect(rewriteSpy).toHaveBeenCalledTimes(1);
|
||||
expect(pool.calls.slice(-3).map((c) => c.text)).toEqual(
|
||||
Array(3).fill('SELECT "name" FROM "apps" WHERE "id" = $1'),
|
||||
);
|
||||
});
|
||||
|
||||
it('normalizes int8 fields on read and primary read rows', async () => {
|
||||
const pool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult(
|
||||
[
|
||||
{
|
||||
uuid: 'session-1',
|
||||
created_at: '1710000000',
|
||||
last_activity: '1710000001',
|
||||
expires_at: '1710000002',
|
||||
revoked_at: null,
|
||||
},
|
||||
],
|
||||
1,
|
||||
[
|
||||
textField('uuid'),
|
||||
int8Field('created_at'),
|
||||
int8Field('last_activity'),
|
||||
int8Field('expires_at'),
|
||||
int8Field('revoked_at'),
|
||||
],
|
||||
),
|
||||
);
|
||||
const client = new PostgresDatabaseClient(postgresConfig(), () => pool);
|
||||
await client.onServerStart();
|
||||
|
||||
await expect(client.read('SELECT * FROM `sessions`')).resolves.toEqual(
|
||||
[
|
||||
{
|
||||
uuid: 'session-1',
|
||||
created_at: 1710000000,
|
||||
last_activity: 1710000001,
|
||||
expires_at: 1710000002,
|
||||
revoked_at: null,
|
||||
},
|
||||
],
|
||||
);
|
||||
await expect(client.pread('SELECT * FROM `sessions`')).resolves.toEqual(
|
||||
[
|
||||
{
|
||||
uuid: 'session-1',
|
||||
created_at: 1710000000,
|
||||
last_activity: 1710000001,
|
||||
expires_at: 1710000002,
|
||||
revoked_at: null,
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects unsafe int8 values instead of losing precision', async () => {
|
||||
const pool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult(
|
||||
[{ id: '9007199254740992' }],
|
||||
1,
|
||||
[int8Field('id')],
|
||||
),
|
||||
);
|
||||
const client = new PostgresDatabaseClient(postgresConfig(), () => pool);
|
||||
await client.onServerStart();
|
||||
|
||||
await expect(client.read('SELECT `id` FROM `sessions`')).rejects.toThrow(
|
||||
'safe integer',
|
||||
);
|
||||
});
|
||||
|
||||
it('normalizes tryHardRead rows returned by a replica', async () => {
|
||||
const primaryPool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult([{ created_at: '1' }], 1, [
|
||||
int8Field('created_at'),
|
||||
]),
|
||||
);
|
||||
const replicaPool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult([{ created_at: '2' }], 1, [
|
||||
int8Field('created_at'),
|
||||
]),
|
||||
);
|
||||
const pools = [primaryPool, replicaPool];
|
||||
let nextPoolIndex = 0;
|
||||
const client = new PostgresDatabaseClient(
|
||||
postgresReplicaConfig(),
|
||||
() => {
|
||||
const pool = pools[nextPoolIndex];
|
||||
nextPoolIndex += 1;
|
||||
if (!pool) throw new Error('unexpected pool factory call');
|
||||
return pool;
|
||||
},
|
||||
);
|
||||
await client.onServerStart();
|
||||
|
||||
await expect(
|
||||
client.tryHardRead('SELECT `created_at` FROM `sessions`'),
|
||||
).resolves.toEqual([{ created_at: 2 }]);
|
||||
});
|
||||
|
||||
it('normalizes tryHardRead rows returned by primary fallback', async () => {
|
||||
const primaryPool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult([{ created_at: '3' }], 1, [
|
||||
int8Field('created_at'),
|
||||
]),
|
||||
);
|
||||
const replicaPool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
queryResult([], 0, [int8Field('created_at')]),
|
||||
);
|
||||
const pools = [primaryPool, replicaPool];
|
||||
let nextPoolIndex = 0;
|
||||
const client = new PostgresDatabaseClient(
|
||||
postgresReplicaConfig(),
|
||||
() => {
|
||||
const pool = pools[nextPoolIndex];
|
||||
nextPoolIndex += 1;
|
||||
if (!pool) throw new Error('unexpected pool factory call');
|
||||
return pool;
|
||||
},
|
||||
);
|
||||
await client.onServerStart();
|
||||
|
||||
await expect(
|
||||
client.tryHardRead('SELECT `created_at` FROM `sessions`'),
|
||||
).resolves.toEqual([{ created_at: 3 }]);
|
||||
});
|
||||
|
||||
it('runs batch writes in order and commits', async () => {
|
||||
const conn = new RecordingPoolClient();
|
||||
const client = new PostgresDatabaseClient(
|
||||
postgresConfig(),
|
||||
() => new RecordingPool(conn),
|
||||
);
|
||||
await client.onServerStart();
|
||||
|
||||
await client.batchWrite([
|
||||
{
|
||||
statement:
|
||||
'UPDATE `user` SET `username` = ? WHERE `id` = ?',
|
||||
values: ['ada', 1],
|
||||
},
|
||||
{
|
||||
statement: 'DELETE FROM `sessions` WHERE `uuid` = ?',
|
||||
values: ['session-1'],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(conn.calls).toEqual([
|
||||
{ text: 'BEGIN', values: undefined },
|
||||
{
|
||||
text: 'UPDATE "user" SET "username" = $1 WHERE "id" = $2',
|
||||
values: ['ada', 1],
|
||||
},
|
||||
{
|
||||
text: 'DELETE FROM "sessions" WHERE "uuid" = $1',
|
||||
values: ['session-1'],
|
||||
},
|
||||
{ text: 'COMMIT', values: undefined },
|
||||
]);
|
||||
expect(conn.released).toBe(true);
|
||||
});
|
||||
|
||||
it('rolls back and releases the pg connection when a batch write fails', async () => {
|
||||
const conn = new RecordingPoolClient('UPDATE "user"');
|
||||
const client = new PostgresDatabaseClient(
|
||||
postgresConfig(),
|
||||
() => new RecordingPool(conn),
|
||||
);
|
||||
await client.onServerStart();
|
||||
|
||||
await expect(
|
||||
client.batchWrite([
|
||||
{
|
||||
statement:
|
||||
'UPDATE `user` SET `username` = ? WHERE `id` = ?',
|
||||
values: ['ada', 1],
|
||||
},
|
||||
]),
|
||||
).rejects.toThrow('forced query failure');
|
||||
|
||||
expect(conn.calls).toEqual([
|
||||
{ text: 'BEGIN', values: undefined },
|
||||
{
|
||||
text: 'UPDATE "user" SET "username" = $1 WHERE "id" = $2',
|
||||
values: ['ada', 1],
|
||||
},
|
||||
{ text: 'ROLLBACK', values: undefined },
|
||||
]);
|
||||
expect(conn.released).toBe(true);
|
||||
});
|
||||
|
||||
it('leaves the pool open through prepare-shutdown; only onServerShutdown closes it', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const pool = new RecordingPool();
|
||||
let ended = false;
|
||||
pool.end = async () => {
|
||||
ended = true;
|
||||
};
|
||||
const client = new PostgresDatabaseClient(
|
||||
postgresConfig(),
|
||||
() => pool,
|
||||
);
|
||||
await client.onServerStart();
|
||||
|
||||
await client.onServerPrepareShutdown();
|
||||
expect(ended).toBe(false);
|
||||
|
||||
// No timer closes the pool on its own — the layers above are
|
||||
// still draining through it for as long as the process drains.
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(ended).toBe(false);
|
||||
|
||||
await client.onServerShutdown();
|
||||
expect(ended).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,302 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { Pool, type PoolConfig, type QueryResult } from 'pg';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
import { Span } from '../../util/span.js';
|
||||
import {
|
||||
AbstractDatabaseClient,
|
||||
type BatchEntry,
|
||||
type WriteResult,
|
||||
} from './DatabaseClient';
|
||||
import { preparePostgresSql } from './preparePostgresSql.js';
|
||||
import { splitPostgresStatements } from './splitPostgresStatements.js';
|
||||
import type { IConfig } from '../../types';
|
||||
|
||||
type PostgresEndpointConfig = {
|
||||
host?: string;
|
||||
port?: number;
|
||||
user?: string;
|
||||
password?: string;
|
||||
database?: string;
|
||||
connectionString?: string;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
export interface PostgresQueryable {
|
||||
query(query: string, values?: unknown[]): Promise<QueryResult>;
|
||||
}
|
||||
|
||||
export interface PostgresPoolClient extends PostgresQueryable {
|
||||
release(): void;
|
||||
}
|
||||
|
||||
export interface PostgresPool extends PostgresQueryable {
|
||||
connect(): Promise<PostgresPoolClient>;
|
||||
end(): Promise<void>;
|
||||
}
|
||||
|
||||
type PostgresPoolFactory = (poolConfig: PoolConfig) => PostgresPool;
|
||||
|
||||
enum Configuration {
|
||||
SINGLE,
|
||||
REPLICA,
|
||||
}
|
||||
|
||||
const POSTGRES_INT8_OID = 20;
|
||||
const PREPARED_SQL_CACHE_SIZE = 1000;
|
||||
const INTEGER_TEXT_PATTERN = /^-?\d+$/u;
|
||||
|
||||
const normalizePostgresInt8 = (
|
||||
value: unknown,
|
||||
columnName: string,
|
||||
): number | null | undefined => {
|
||||
if (value === null || value === undefined) return value;
|
||||
|
||||
const parsed =
|
||||
typeof value === 'bigint'
|
||||
? Number(value)
|
||||
: typeof value === 'number'
|
||||
? value
|
||||
: typeof value === 'string' && INTEGER_TEXT_PATTERN.test(value)
|
||||
? Number(value)
|
||||
: Number.NaN;
|
||||
|
||||
if (!Number.isSafeInteger(parsed)) {
|
||||
throw new Error(
|
||||
`[postgres] int8 column ${columnName} is outside JavaScript's safe integer range`,
|
||||
);
|
||||
}
|
||||
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const normalizePostgresRows = (
|
||||
result: QueryResult,
|
||||
): Record<string, unknown>[] => {
|
||||
const int8Fields = result.fields.filter(
|
||||
(field) => field.dataTypeID === POSTGRES_INT8_OID,
|
||||
);
|
||||
if (int8Fields.length === 0) {
|
||||
return result.rows as Record<string, unknown>[];
|
||||
}
|
||||
|
||||
return result.rows.map((row) => {
|
||||
const normalized: Record<string, unknown> = { ...row };
|
||||
for (const field of int8Fields) {
|
||||
normalized[field.name] = normalizePostgresInt8(
|
||||
normalized[field.name],
|
||||
field.name,
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
});
|
||||
};
|
||||
|
||||
export const mapPostgresWriteResult = (result: QueryResult): WriteResult => {
|
||||
const affectedRows = result.rowCount ?? 0;
|
||||
const rowId = result.rows[0]?.id;
|
||||
const insertId =
|
||||
typeof rowId === 'bigint'
|
||||
? rowId
|
||||
: typeof rowId === 'number'
|
||||
? rowId
|
||||
: typeof rowId === 'string' && rowId !== ''
|
||||
? Number(rowId)
|
||||
: 0;
|
||||
const normalizedInsertId =
|
||||
typeof insertId === 'number' && Number.isNaN(insertId) ? 0 : insertId;
|
||||
|
||||
return {
|
||||
insertId: normalizedInsertId,
|
||||
affectedRows,
|
||||
anyRowsAffected: affectedRows > 0,
|
||||
};
|
||||
};
|
||||
|
||||
export class PostgresDatabaseClient extends AbstractDatabaseClient {
|
||||
override readonly engineName = 'postgres';
|
||||
|
||||
private primaryPool!: PostgresPool;
|
||||
private replicaPool!: PostgresPool;
|
||||
private configuration = Configuration.SINGLE;
|
||||
private shutdownStarted = false;
|
||||
/** Query text as written → its Postgres rewrite, which is pure. */
|
||||
private readonly preparedSql = new BoundedTtlMap<string, string>({
|
||||
maxEntries: PREPARED_SQL_CACHE_SIZE,
|
||||
});
|
||||
|
||||
constructor(
|
||||
config: IConfig,
|
||||
private readonly poolFactory: PostgresPoolFactory = (poolConfig) =>
|
||||
new Pool(poolConfig) as unknown as PostgresPool,
|
||||
) {
|
||||
super(config);
|
||||
}
|
||||
|
||||
override async onServerStart(): Promise<void> {
|
||||
const dbConf = this.config.database!;
|
||||
|
||||
this.primaryPool = this.createPool(dbConf);
|
||||
await this.primaryPool.query('SELECT 1');
|
||||
console.log('[postgres] connected to primary');
|
||||
|
||||
if (dbConf.replica) {
|
||||
this.replicaPool = this.createPool(dbConf.replica);
|
||||
await this.replicaPool.query('SELECT 1');
|
||||
this.configuration = Configuration.REPLICA;
|
||||
console.log('[postgres] connected to read-replica');
|
||||
} else {
|
||||
this.replicaPool = this.primaryPool;
|
||||
this.configuration = Configuration.SINGLE;
|
||||
}
|
||||
|
||||
await this.applyMigrationPaths(
|
||||
() => this.primaryPool.connect(),
|
||||
splitPostgresStatements,
|
||||
{ transactional: true },
|
||||
);
|
||||
}
|
||||
|
||||
override async onServerPrepareShutdown(): Promise<void> {
|
||||
if (this.shutdownStarted) return;
|
||||
this.shutdownStarted = true;
|
||||
|
||||
// Pools stay open; onServerShutdown closes them after the layers above
|
||||
// have drained through them.
|
||||
console.log('[postgres] entering drain mode');
|
||||
}
|
||||
|
||||
override async onServerShutdown(): Promise<void> {
|
||||
await this.closeCurrentPools();
|
||||
}
|
||||
|
||||
override quoteIdentifier(identifier: string): string {
|
||||
return identifier
|
||||
.split('.')
|
||||
.map((part) => {
|
||||
if (part === '*') return part;
|
||||
return `"${part.replaceAll('"', '""')}"`;
|
||||
})
|
||||
.join('.');
|
||||
}
|
||||
|
||||
override booleanLiteral(value: boolean): string {
|
||||
return value ? 'TRUE' : 'FALSE';
|
||||
}
|
||||
|
||||
override booleanValue(value: boolean): boolean {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Span('db.read', (query: string) => ({ 'db.statement': query }))
|
||||
override async read(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const result = await this.query(this.replicaPool, query, params);
|
||||
return normalizePostgresRows(result);
|
||||
}
|
||||
|
||||
@Span('db.pread', (query: string) => ({ 'db.statement': query }))
|
||||
override async pread(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const result = await this.query(this.primaryPool, query, params);
|
||||
return normalizePostgresRows(result);
|
||||
}
|
||||
|
||||
@Span('db.write', (query: string) => ({ 'db.statement': query }))
|
||||
override async write(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<WriteResult> {
|
||||
const result = await this.query(this.primaryPool, query, params);
|
||||
return mapPostgresWriteResult(result);
|
||||
}
|
||||
|
||||
@Span('db.batchWrite', (entries: unknown[]) => ({
|
||||
'db.batch_size': entries.length,
|
||||
}))
|
||||
override async batchWrite(entries: BatchEntry[]): Promise<void> {
|
||||
if (entries.length === 0) return;
|
||||
|
||||
const conn = await this.primaryPool.connect();
|
||||
try {
|
||||
await conn.query('BEGIN');
|
||||
try {
|
||||
for (const { statement, values } of entries) {
|
||||
await this.query(conn, statement, values);
|
||||
}
|
||||
await conn.query('COMMIT');
|
||||
} catch (err) {
|
||||
await conn.query('ROLLBACK').catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
}
|
||||
|
||||
protected override hasReadReplica(): boolean {
|
||||
return this.configuration === Configuration.REPLICA;
|
||||
}
|
||||
|
||||
private createPool(dbConf: PostgresEndpointConfig): PostgresPool {
|
||||
const connectionString = dbConf.connectionString ?? dbConf.url;
|
||||
if (connectionString) {
|
||||
return this.poolFactory({
|
||||
connectionString,
|
||||
max: 30,
|
||||
});
|
||||
}
|
||||
|
||||
return this.poolFactory({
|
||||
host: dbConf.host ?? '127.0.0.1',
|
||||
port: dbConf.port ?? 5432,
|
||||
user: dbConf.user ?? 'postgres',
|
||||
password: dbConf.password ?? '',
|
||||
database: dbConf.database ?? 'puter',
|
||||
max: 30,
|
||||
});
|
||||
}
|
||||
|
||||
private async query(
|
||||
target: PostgresQueryable,
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<QueryResult> {
|
||||
let text = this.preparedSql.get(query);
|
||||
if (text === undefined) {
|
||||
text = preparePostgresSql(query).text;
|
||||
this.preparedSql.set(query, text);
|
||||
}
|
||||
return target.query(text, params);
|
||||
}
|
||||
|
||||
private async closeCurrentPools(): Promise<void> {
|
||||
const tasks: Promise<void>[] = [];
|
||||
if (this.primaryPool) tasks.push(this.primaryPool.end());
|
||||
if (this.replicaPool && this.replicaPool !== this.primaryPool) {
|
||||
tasks.push(this.replicaPool.end());
|
||||
}
|
||||
await Promise.all(tasks);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
/*
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
@@ -18,20 +18,10 @@
|
||||
*/
|
||||
|
||||
import { metrics } from '@opentelemetry/api';
|
||||
import {
|
||||
POOL_ACQUIRE_TIMEOUT,
|
||||
isNeverSentError,
|
||||
isRetriableError,
|
||||
isRolledBackError,
|
||||
} from './retriableErrors.js';
|
||||
|
||||
const DEFAULT_MAX_QUEUE_SIZE = 1000;
|
||||
const DEFAULT_FAILURE_THRESHOLD = 5;
|
||||
const DEFAULT_COOLDOWN_MS = 5_000;
|
||||
const DEFAULT_ACQUIRE_TIMEOUT_MS = 5_000;
|
||||
const ACQUIRE_ATTEMPTS = 3;
|
||||
const ITEM_RETRY_ATTEMPTS = 2;
|
||||
const RETRY_BASE_BACKOFF_MS = 100;
|
||||
const FALLBACK_RETRY_CONCURRENCY = 8;
|
||||
|
||||
const meter = metrics.getMeter('puter-backend');
|
||||
@@ -63,85 +53,33 @@ const fallbackItemFailuresCounter = meter.createCounter(
|
||||
'Per-item failures observed during SQLBatcher per-item retry',
|
||||
},
|
||||
);
|
||||
const fallbackItemRetriesCounter = meter.createCounter(
|
||||
'sql_batcher.fallback.item_retries',
|
||||
{
|
||||
description:
|
||||
'Transient per-item failures retried during SQLBatcher fallback',
|
||||
},
|
||||
);
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export class SQLBatcher {
|
||||
dbPool;
|
||||
maxTimeInQueue;
|
||||
maxInFlightImmediate;
|
||||
maxBatchSize;
|
||||
maxQueueSize;
|
||||
failureThreshold;
|
||||
cooldownMs;
|
||||
poolLabel;
|
||||
readOnly;
|
||||
acquireTimeoutMs;
|
||||
queue = [];
|
||||
#flushTimer = null;
|
||||
#flushTimerIsImmediate = false;
|
||||
#inFlight;
|
||||
timeouts = [];
|
||||
#consecutiveFailures = 0;
|
||||
#lastFailureAt = 0;
|
||||
#metricAttrs;
|
||||
|
||||
/**
|
||||
* @param {object} dbPool Mysql2 pool
|
||||
* @param {object} [opts]
|
||||
* @param {number} [opts.maxTimeInQueue] Ms an item may wait before flush
|
||||
* @param {number} [opts.maxInFlightImmediate] While fewer flushes than this
|
||||
* are in flight, a lone item flushes on the next turn instead of waiting
|
||||
* maxTimeInQueue; past it, items wait and coalesce to spare connections
|
||||
* @param {{ count: number }} [opts.inFlight] In-flight flush counter the
|
||||
* cap reads; batchers sharing a pool can share one
|
||||
* @param {number} [opts.maxBatchSize] Items coalesced per flush
|
||||
* @param {number} [opts.maxQueueSize] Drop-oldest high-water mark
|
||||
* @param {number} [opts.failureThreshold] Consecutive failures to open the
|
||||
* breaker
|
||||
* @param {number} [opts.cooldownMs] Breaker open duration after last
|
||||
* failure
|
||||
* @param {'primary' | 'replica'} [opts.poolLabel] Role label on metrics; in
|
||||
* single-node setups the 'replica' batcher shares the primary pool, so
|
||||
* this reflects the read/write role rather than a physical instance
|
||||
* @param {boolean} [opts.readOnly] This batcher only ever carries SELECTs,
|
||||
* so any transient failure is safe to retry
|
||||
* @param {number} [opts.acquireTimeoutMs] Max wait for a pooled connection;
|
||||
* 0 disables the bound
|
||||
*/
|
||||
constructor(
|
||||
dbPool,
|
||||
{
|
||||
maxTimeInQueue = 20,
|
||||
maxInFlightImmediate = 0,
|
||||
inFlight = { count: 0 },
|
||||
maxBatchSize = 50,
|
||||
maxQueueSize = DEFAULT_MAX_QUEUE_SIZE,
|
||||
failureThreshold = DEFAULT_FAILURE_THRESHOLD,
|
||||
cooldownMs = DEFAULT_COOLDOWN_MS,
|
||||
poolLabel = 'primary',
|
||||
readOnly = false,
|
||||
acquireTimeoutMs = DEFAULT_ACQUIRE_TIMEOUT_MS,
|
||||
} = {},
|
||||
maxTimeInQueue = 20,
|
||||
maxBatchSize = 50,
|
||||
maxQueueSize = DEFAULT_MAX_QUEUE_SIZE,
|
||||
failureThreshold = DEFAULT_FAILURE_THRESHOLD,
|
||||
cooldownMs = DEFAULT_COOLDOWN_MS,
|
||||
) {
|
||||
this.dbPool = dbPool;
|
||||
this.maxTimeInQueue = maxTimeInQueue;
|
||||
this.maxInFlightImmediate = maxInFlightImmediate;
|
||||
this.#inFlight = inFlight;
|
||||
this.maxBatchSize = maxBatchSize;
|
||||
this.maxQueueSize = maxQueueSize;
|
||||
this.failureThreshold = failureThreshold;
|
||||
this.cooldownMs = cooldownMs;
|
||||
this.poolLabel = poolLabel;
|
||||
this.readOnly = readOnly;
|
||||
this.acquireTimeoutMs = acquireTimeoutMs;
|
||||
this.#metricAttrs = { pool: poolLabel };
|
||||
}
|
||||
|
||||
async execute(sql, values) {
|
||||
@@ -152,13 +90,9 @@ export class SQLBatcher {
|
||||
return this;
|
||||
}
|
||||
|
||||
// The public error is deliberately opaque (no SQL, no internals), but
|
||||
// `reason` distinguishes the load-shed path for logs and callers:
|
||||
// breakerOpen | queueOverflow | connAcquire.
|
||||
#createPublicBatchError(reason) {
|
||||
#createPublicBatchError() {
|
||||
const error = new Error('Database operation failed');
|
||||
error.code = 'dbBatchFailed';
|
||||
error.reason = reason;
|
||||
return error;
|
||||
}
|
||||
|
||||
@@ -172,8 +106,8 @@ export class SQLBatcher {
|
||||
|
||||
async query(sql, values) {
|
||||
if (this.#isBreakerOpen()) {
|
||||
enqueueRejectedCounter.add(1, this.#metricAttrs);
|
||||
throw this.#createPublicBatchError('breakerOpen');
|
||||
enqueueRejectedCounter.add(1);
|
||||
throw this.#createPublicBatchError();
|
||||
}
|
||||
|
||||
const { promise, resolve, reject } = Promise.withResolvers();
|
||||
@@ -183,8 +117,8 @@ export class SQLBatcher {
|
||||
// to have already exceeded any caller-side timeout anyway.
|
||||
while (this.queue.length >= this.maxQueueSize) {
|
||||
const dropped = this.queue.shift();
|
||||
dropped.reject(this.#createPublicBatchError('queueOverflow'));
|
||||
enqueueDroppedCounter.add(1, this.#metricAttrs);
|
||||
dropped.reject(this.#createPublicBatchError());
|
||||
enqueueDroppedCounter.add(1);
|
||||
}
|
||||
|
||||
this.queue.push({
|
||||
@@ -197,141 +131,55 @@ export class SQLBatcher {
|
||||
|
||||
if (this.queue.length >= this.maxBatchSize) {
|
||||
this.flush(this.queue.splice(0, this.maxBatchSize));
|
||||
} else if (this.#flushTimer === null) {
|
||||
this.#armFlushTimer();
|
||||
} else if (this.queue.length === 1) {
|
||||
this.timeouts.push(
|
||||
setTimeout(() => {
|
||||
this.flush(this.queue.splice(0, this.queue.length));
|
||||
}, this.maxTimeInQueue),
|
||||
);
|
||||
}
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
// A lone flush holds a pooled connection for a full round trip, so only
|
||||
// the first maxInFlightImmediate flushes skip the wait.
|
||||
#armFlushTimer() {
|
||||
const run = () => {
|
||||
this.#flushTimer = null;
|
||||
this.flush(this.queue.splice(0, this.queue.length));
|
||||
};
|
||||
this.#flushTimerIsImmediate =
|
||||
this.#inFlight.count < this.maxInFlightImmediate;
|
||||
this.#flushTimer = this.#flushTimerIsImmediate
|
||||
? setImmediate(run)
|
||||
: setTimeout(run, this.maxTimeInQueue);
|
||||
}
|
||||
|
||||
#clearFlushTimer() {
|
||||
if (this.#flushTimer === null) return;
|
||||
if (this.#flushTimerIsImmediate) clearImmediate(this.#flushTimer);
|
||||
else clearTimeout(this.#flushTimer);
|
||||
this.#flushTimer = null;
|
||||
}
|
||||
|
||||
// Bounded wait for a pooled connection. Without a bound, a stalled
|
||||
// database turns every flush into an indefinite hang — nothing fails,
|
||||
// so neither the breaker nor callers' own timeouts ever engage.
|
||||
#getConnectionWithTimeout() {
|
||||
const acquire = this.dbPool.promise().getConnection();
|
||||
if (!this.acquireTimeoutMs) return acquire;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
// A connection that arrives late must go back to the pool.
|
||||
acquire.then(
|
||||
(conn) => conn.release(),
|
||||
() => {},
|
||||
);
|
||||
const error = new Error(
|
||||
'Timed out acquiring database connection',
|
||||
);
|
||||
error.code = POOL_ACQUIRE_TIMEOUT;
|
||||
reject(error);
|
||||
}, this.acquireTimeoutMs);
|
||||
|
||||
acquire.then(
|
||||
(conn) => {
|
||||
if (timedOut) return;
|
||||
clearTimeout(timer);
|
||||
resolve(conn);
|
||||
},
|
||||
(err) => {
|
||||
if (timedOut) return;
|
||||
clearTimeout(timer);
|
||||
reject(err);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Acquisition failures never sent a statement, so retrying is always
|
||||
// safe regardless of what the batch contains.
|
||||
async #acquireConnection() {
|
||||
let lastError;
|
||||
for (let attempt = 1; attempt <= ACQUIRE_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
return await this.#getConnectionWithTimeout();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt < ACQUIRE_ATTEMPTS) {
|
||||
await sleep(RETRY_BASE_BACKOFF_MS * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
async flush(batch) {
|
||||
this.#clearFlushTimer();
|
||||
const timeout = this.timeouts.shift();
|
||||
if (timeout && !timeout._destroyed) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
if (batch.length === 0) return;
|
||||
|
||||
this.#inFlight.count++;
|
||||
try {
|
||||
await this.#flushBatch(batch);
|
||||
} finally {
|
||||
this.#inFlight.count--;
|
||||
}
|
||||
}
|
||||
const query = `${batch.map((b) => b.sql.replace(/;+\s*$/, '')).join(';')}; SELECT 1`; // SELECT 1 forces mysql2 to return array
|
||||
const values = batch.map((b) => b.values ?? []).flat();
|
||||
|
||||
async #flushBatch(batch) {
|
||||
let connection;
|
||||
try {
|
||||
connection = await this.#acquireConnection();
|
||||
connection = await this.dbPool.promise().getConnection();
|
||||
} catch (error) {
|
||||
this.#recordFlushFailure();
|
||||
this.#consecutiveFailures++;
|
||||
this.#lastFailureAt = Date.now();
|
||||
flushFailureCounter.add(1);
|
||||
console.warn(
|
||||
'SQLBatcher could not acquire connection for flush:',
|
||||
error,
|
||||
);
|
||||
for (const b of batch) {
|
||||
b.reject(this.#createPublicBatchError('connAcquire'));
|
||||
b.reject(this.#createPublicBatchError());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (batch.length === 1 && !this.readOnly) {
|
||||
await this.#flushLoneWrite(batch[0], connection);
|
||||
return;
|
||||
}
|
||||
|
||||
const query = `${batch.map((b) => b.sql.replace(/;+\s*$/, '')).join(';')}; SELECT 1`; // SELECT 1 forces mysql2 to return array
|
||||
const values = batch.map((b) => b.values ?? []).flat();
|
||||
|
||||
// Run the coalesced multi-statement inside an explicit transaction so
|
||||
// a single bad statement (e.g. a duplicate-key INSERT) rolls back the
|
||||
// whole batch atomically, leaving us free to re-run each item
|
||||
// individually below. Without this, MySQL would commit every
|
||||
// statement up to the failure point and a per-item retry would
|
||||
// misreport already-committed inserts as duplicate-key failures.
|
||||
//
|
||||
// A read-only batch has nothing to roll back, and the wrapper is not
|
||||
// free: BEGIN and COMMIT are round trips of their own, which triples
|
||||
// the cost of a read against a pool in another region.
|
||||
const wrapInTransaction = !this.readOnly;
|
||||
let batchSucceeded = false;
|
||||
try {
|
||||
if (wrapInTransaction) await connection.beginTransaction();
|
||||
await connection.beginTransaction();
|
||||
const [results, fields] = await connection.query(query, values);
|
||||
if (wrapInTransaction) await connection.commit();
|
||||
await connection.commit();
|
||||
batchSucceeded = true;
|
||||
this.#consecutiveFailures = 0;
|
||||
for (let i = 0; i < batch.length; i++) {
|
||||
@@ -339,12 +187,10 @@ export class SQLBatcher {
|
||||
b.resolve([results[i], fields?.[i]]);
|
||||
}
|
||||
} catch (batchError) {
|
||||
if (wrapInTransaction) {
|
||||
try {
|
||||
await connection.rollback();
|
||||
} catch (rollbackError) {
|
||||
console.warn('SQLBatcher rollback failed:', rollbackError);
|
||||
}
|
||||
try {
|
||||
await connection.rollback();
|
||||
} catch (rollbackError) {
|
||||
console.warn('SQLBatcher rollback failed:', rollbackError);
|
||||
}
|
||||
console.warn(
|
||||
'SQLBatcher batch failed; retrying items individually:',
|
||||
@@ -356,13 +202,12 @@ export class SQLBatcher {
|
||||
|
||||
if (batchSucceeded) return;
|
||||
|
||||
// Per-item fallback. Nothing from the batch is live — it was rolled
|
||||
// back, or carried only SELECTs — so re-running each item
|
||||
// independently produces clean success/failure outcomes for each
|
||||
// caller. Concurrency is capped to avoid briefly saturating the pool
|
||||
// when a large batch fails.
|
||||
flushFailureCounter.add(1, this.#metricAttrs);
|
||||
fallbackInvocationsCounter.add(1, this.#metricAttrs);
|
||||
// Per-item fallback. The transaction was rolled back so no statement
|
||||
// committed; re-running each item independently produces clean
|
||||
// success/failure outcomes for each caller. Concurrency is capped to
|
||||
// avoid briefly saturating the pool when a large batch fails.
|
||||
flushFailureCounter.add(1);
|
||||
fallbackInvocationsCounter.add(1);
|
||||
|
||||
const settled = new Array(batch.length);
|
||||
let cursor = 0;
|
||||
@@ -371,7 +216,17 @@ export class SQLBatcher {
|
||||
async () => {
|
||||
while (cursor < batch.length) {
|
||||
const i = cursor++;
|
||||
settled[i] = await this.#runItem(batch[i]);
|
||||
const b = batch[i];
|
||||
try {
|
||||
settled[i] = {
|
||||
ok: true,
|
||||
value: await this.dbPool
|
||||
.promise()
|
||||
.query(b.sql, b.values ?? []),
|
||||
};
|
||||
} catch (error) {
|
||||
settled[i] = { ok: false, error };
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -391,7 +246,7 @@ export class SQLBatcher {
|
||||
}
|
||||
}
|
||||
if (failureCount > 0) {
|
||||
fallbackItemFailuresCounter.add(failureCount, this.#metricAttrs);
|
||||
fallbackItemFailuresCounter.add(failureCount);
|
||||
}
|
||||
|
||||
// Only escalate the breaker when the database itself looks unhealthy
|
||||
@@ -404,76 +259,4 @@ export class SQLBatcher {
|
||||
this.#consecutiveFailures++;
|
||||
}
|
||||
}
|
||||
|
||||
// A single statement is atomic under autocommit, so it skips BEGIN/COMMIT
|
||||
// and their round trips. With no transaction to roll back, a failure is
|
||||
// not re-run wholesale; only the per-item retry policy applies.
|
||||
async #flushLoneWrite(item, connection) {
|
||||
const r = await this.#runItem(item, connection);
|
||||
if (r.ok) {
|
||||
this.#consecutiveFailures = 0;
|
||||
item.resolve(r.value);
|
||||
return;
|
||||
}
|
||||
// An error the server answered with (duplicate key, bad SQL) means it
|
||||
// is up; only failing to reach it counts toward the breaker.
|
||||
if (isRetriableError(r.error)) {
|
||||
this.#recordFlushFailure();
|
||||
} else {
|
||||
this.#consecutiveFailures = 0;
|
||||
flushFailureCounter.add(1, this.#metricAttrs);
|
||||
}
|
||||
item.reject(r.error);
|
||||
}
|
||||
|
||||
#recordFlushFailure() {
|
||||
this.#consecutiveFailures++;
|
||||
this.#lastFailureAt = Date.now();
|
||||
flushFailureCounter.add(1, this.#metricAttrs);
|
||||
}
|
||||
|
||||
// Run one item on its own, retrying transient failures with backoff.
|
||||
// A read-only batcher may retry anything transient; a batcher that
|
||||
// carries writes only retries failures where the statement provably
|
||||
// did not apply — either it never reached the server, or the server
|
||||
// rolled it back itself. A write that died mid-flight may have
|
||||
// committed, and re-running that one would double-apply.
|
||||
//
|
||||
// Lock contention lands in the second group and is worth retrying rather
|
||||
// than surfacing: an item is a single statement, so a deadlock victim has
|
||||
// been fully undone, and the caller sees an unhandled 500 for what the
|
||||
// database is telling us to just run again.
|
||||
async #runItem(b, connection) {
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
if (!connection) {
|
||||
try {
|
||||
connection = await this.#acquireConnection();
|
||||
} catch (error) {
|
||||
return { ok: false, error };
|
||||
}
|
||||
}
|
||||
let error;
|
||||
try {
|
||||
return {
|
||||
ok: true,
|
||||
value: await connection.query(b.sql, b.values ?? []),
|
||||
};
|
||||
} catch (e) {
|
||||
error = e;
|
||||
} finally {
|
||||
connection.release();
|
||||
connection = undefined;
|
||||
}
|
||||
const canRetry = this.readOnly
|
||||
? isRetriableError(error) || isRolledBackError(error)
|
||||
: isNeverSentError(error) || isRolledBackError(error);
|
||||
if (!canRetry || attempt >= ITEM_RETRY_ATTEMPTS) {
|
||||
return { ok: false, error };
|
||||
}
|
||||
attempt++;
|
||||
fallbackItemRetriesCounter.add(1, this.#metricAttrs);
|
||||
await sleep(RETRY_BASE_BACKOFF_MS * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,717 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { SQLBatcher } from './SQLBatcher.js';
|
||||
|
||||
const makeError = (code: string, message = code): Error & { code: string } => {
|
||||
const error = new Error(message) as Error & { code: string };
|
||||
error.code = code;
|
||||
return error;
|
||||
};
|
||||
|
||||
interface FakeConnection {
|
||||
beginTransaction: ReturnType<typeof vi.fn>;
|
||||
query: ReturnType<typeof vi.fn>;
|
||||
commit: ReturnType<typeof vi.fn>;
|
||||
rollback: ReturnType<typeof vi.fn>;
|
||||
release: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
// The batch flush sends one coalesced multi-statement (always suffixed
|
||||
// `; SELECT 1`); fallback items send their original single statement.
|
||||
const isBatchQuery = (sql: string) => sql.endsWith('; SELECT 1');
|
||||
|
||||
const makeConnection = (
|
||||
onQuery: (sql: string, values: unknown[]) => unknown,
|
||||
): FakeConnection => ({
|
||||
beginTransaction: vi.fn(async () => {}),
|
||||
query: vi.fn(async (sql: string, values: unknown[]) => onQuery(sql, values)),
|
||||
commit: vi.fn(async () => {}),
|
||||
rollback: vi.fn(async () => {}),
|
||||
release: vi.fn(),
|
||||
});
|
||||
|
||||
const makePool = (connection: FakeConnection | (() => Promise<FakeConnection>)) => {
|
||||
const getConnection = vi.fn(async () =>
|
||||
typeof connection === 'function' ? connection() : connection,
|
||||
);
|
||||
return {
|
||||
pool: { promise: () => ({ getConnection }) },
|
||||
getConnection,
|
||||
};
|
||||
};
|
||||
|
||||
// Happy-path onQuery: batch returns one result row-set per statement plus
|
||||
// the trailing SELECT 1 row-set.
|
||||
const happyBatch = (sql: string) => {
|
||||
if (!isBatchQuery(sql)) throw new Error('unexpected fallback query');
|
||||
const statements = sql.split(';').length - 1;
|
||||
return [
|
||||
Array.from({ length: statements + 1 }, (_, i) => [{ n: i }]),
|
||||
undefined,
|
||||
];
|
||||
};
|
||||
|
||||
// Each query on this connection waits until the test releases it, so its
|
||||
// flush stays in flight.
|
||||
const makeHeldConnection = () => {
|
||||
const held: Array<() => void> = [];
|
||||
const conn = makeConnection(
|
||||
(sql) =>
|
||||
new Promise((resolve) => {
|
||||
held.push(() => resolve(happyBatch(sql)));
|
||||
}),
|
||||
);
|
||||
const releaseAll = () => {
|
||||
for (const release of held.splice(0)) release();
|
||||
};
|
||||
const sent = () => conn.query.mock.calls.map(([sql]) => sql);
|
||||
return { conn, releaseAll, sent };
|
||||
};
|
||||
|
||||
const fakeFlushTimers = () =>
|
||||
vi.useFakeTimers({
|
||||
toFake: [
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
|
||||
describe('SQLBatcher flush timing', () => {
|
||||
it('flushes a lone query on the next turn while under the in-flight cap', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 60_000,
|
||||
maxInFlightImmediate: 2,
|
||||
});
|
||||
|
||||
const pending = batcher.query('SELECT a', []);
|
||||
expect(getConnection).not.toHaveBeenCalled();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(getConnection).toHaveBeenCalledTimes(1);
|
||||
expect((await pending)[0]).toEqual([{ n: 0 }]);
|
||||
});
|
||||
|
||||
it('holds a lone query for maxTimeInQueue without an in-flight cap', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
});
|
||||
|
||||
const pending = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
expect(getConnection).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(getConnection).toHaveBeenCalledTimes(1);
|
||||
expect((await pending)[0]).toEqual([{ n: 0 }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('batches queries from the same turn and sends later ones separately', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxInFlightImmediate: 8,
|
||||
});
|
||||
|
||||
const sameTurn = [
|
||||
batcher.query('SELECT a', []),
|
||||
batcher.query('SELECT b', []),
|
||||
batcher.query('SELECT c', []),
|
||||
];
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const nextTurn = batcher.query('SELECT d', []);
|
||||
const results = await Promise.all([...sameTurn, nextTurn]);
|
||||
|
||||
expect(results.map((r) => r[0])).toEqual([
|
||||
[{ n: 0 }],
|
||||
[{ n: 1 }],
|
||||
[{ n: 2 }],
|
||||
[{ n: 0 }],
|
||||
]);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
expect(conn.query.mock.calls.map(([sql]) => sql)).toEqual([
|
||||
'SELECT a;SELECT b;SELECT c; SELECT 1',
|
||||
'SELECT d; SELECT 1',
|
||||
]);
|
||||
});
|
||||
|
||||
it('gives a query enqueued during an in-flight flush its own flush while under the cap', async () => {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxInFlightImmediate: 2,
|
||||
});
|
||||
const turn = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await turn();
|
||||
const b = batcher.query('SELECT b', []);
|
||||
await turn();
|
||||
expect(sent()).toEqual(['SELECT a; SELECT 1', 'SELECT b; SELECT 1']);
|
||||
|
||||
releaseAll();
|
||||
expect((await a)[0]).toEqual([{ n: 0 }]);
|
||||
expect((await b)[0]).toEqual([{ n: 0 }]);
|
||||
});
|
||||
|
||||
it('holds a lone query for maxTimeInQueue at the in-flight cap and coalesces what follows', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toEqual(['SELECT a; SELECT 1']);
|
||||
|
||||
const b = batcher.query('SELECT b', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
const c = batcher.query('SELECT c', []);
|
||||
expect(sent()).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT b;SELECT c; SELECT 1',
|
||||
]);
|
||||
|
||||
releaseAll();
|
||||
expect((await a)[0]).toEqual([{ n: 0 }]);
|
||||
expect((await b)[0]).toEqual([{ n: 0 }]);
|
||||
expect((await c)[0]).toEqual([{ n: 1 }]);
|
||||
|
||||
// Settled flushes free the cap again.
|
||||
const d = batcher.query('SELECT d', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toHaveLength(3);
|
||||
releaseAll();
|
||||
expect((await d)[0]).toEqual([{ n: 0 }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('shares the in-flight cap between batchers given one counter', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool } = makePool(conn);
|
||||
const inFlight = { count: 0 };
|
||||
const opts = {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
inFlight,
|
||||
};
|
||||
const first = new SQLBatcher(pool, opts);
|
||||
const second = new SQLBatcher(pool, opts);
|
||||
const own = new SQLBatcher(pool, { ...opts, inFlight: undefined });
|
||||
|
||||
const a = first.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
const b = second.query('SELECT b', []);
|
||||
const c = own.query('SELECT c', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT c; SELECT 1',
|
||||
]);
|
||||
await vi.advanceTimersByTimeAsync(30);
|
||||
expect(sent()).toHaveLength(3);
|
||||
expect(sent()[2]).toBe('SELECT b; SELECT 1');
|
||||
|
||||
releaseAll();
|
||||
await Promise.all([a, b, c]);
|
||||
expect(inFlight.count).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('counts a flush as in flight until its per-item fallback settles', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const held: Array<() => void> = [];
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ECONNRESET');
|
||||
return new Promise((resolve) => {
|
||||
held.push(() => resolve([[{ ok: sql }], undefined]));
|
||||
});
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const sent = () => conn.query.mock.calls.map(([sql]) => sql);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
acquireTimeoutMs: 0,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
// The batch failed; the per-item retry of `a` is still running.
|
||||
expect(sent()).toEqual(['SELECT a; SELECT 1', 'SELECT a']);
|
||||
|
||||
const b = batcher.query('SELECT b', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
expect(sent()).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT a',
|
||||
'SELECT b; SELECT 1',
|
||||
'SELECT b',
|
||||
]);
|
||||
|
||||
for (const release of held.splice(0)) release();
|
||||
expect((await a)[0]).toEqual([{ ok: 'SELECT a' }]);
|
||||
expect((await b)[0]).toEqual([{ ok: 'SELECT b' }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('flushes a full batch at once past the cap and cancels the armed wait', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxBatchSize: 2,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
acquireTimeoutMs: 0,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
const b = batcher.query('SELECT b', []);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
const c = batcher.query('SELECT c', []);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
// A later lone query waits its own full maxTimeInQueue.
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
const d = batcher.query('SELECT d', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(getConnection).toHaveBeenCalledTimes(3);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT b;SELECT c; SELECT 1',
|
||||
'SELECT d; SELECT 1',
|
||||
]);
|
||||
|
||||
releaseAll();
|
||||
await Promise.all([a, b, c, d]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('cancels the pending next-turn flush when a batch fills up', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxBatchSize: 2,
|
||||
readOnly: true,
|
||||
maxInFlightImmediate: 8,
|
||||
acquireTimeoutMs: 0,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
const b = batcher.query('SELECT b', []);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
const c = batcher.query('SELECT c', []);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
await vi.runAllTimersAsync();
|
||||
await Promise.all([a, b, c]);
|
||||
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves no timer armed after a failed flush, a drop or a breaker rejection', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const getConnection = vi.fn(async () => {
|
||||
throw makeError('ECONNREFUSED');
|
||||
});
|
||||
const batcher = new SQLBatcher(
|
||||
{ promise: () => ({ getConnection }) },
|
||||
{
|
||||
readOnly: true,
|
||||
maxQueueSize: 1,
|
||||
failureThreshold: 1,
|
||||
cooldownMs: 60_000,
|
||||
maxInFlightImmediate: 8,
|
||||
},
|
||||
);
|
||||
|
||||
const dropped = expect(
|
||||
batcher.query('SELECT a', []),
|
||||
).rejects.toMatchObject({ reason: 'queueOverflow' });
|
||||
const failed = expect(
|
||||
batcher.query('SELECT b', []),
|
||||
).rejects.toMatchObject({ reason: 'connAcquire' });
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
await vi.runAllTimersAsync();
|
||||
await dropped;
|
||||
await failed;
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
await expect(batcher.query('SELECT c', [])).rejects.toMatchObject({
|
||||
reason: 'breakerOpen',
|
||||
});
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(batcher.queue).toHaveLength(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SQLBatcher', () => {
|
||||
it('resolves each batched item with its own result', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 });
|
||||
|
||||
const [a, b] = await Promise.all([
|
||||
batcher.query('SELECT a', []),
|
||||
batcher.query('SELECT b', []),
|
||||
]);
|
||||
expect(a[0]).toEqual([{ n: 0 }]);
|
||||
expect(b[0]).toEqual([{ n: 1 }]);
|
||||
expect(conn.beginTransaction).toHaveBeenCalledTimes(1);
|
||||
expect(conn.commit).toHaveBeenCalledTimes(1);
|
||||
expect(conn.release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('skips the transaction wrapper on a readOnly batcher', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
readOnly: true,
|
||||
});
|
||||
|
||||
const [a, b] = await Promise.all([
|
||||
batcher.query('SELECT a', []),
|
||||
batcher.query('SELECT b', []),
|
||||
]);
|
||||
expect(a[0]).toEqual([{ n: 0 }]);
|
||||
expect(b[0]).toEqual([{ n: 1 }]);
|
||||
// BEGIN and COMMIT are round trips; a SELECT-only batch has nothing to
|
||||
// roll back, so it must not pay for them.
|
||||
expect(conn.beginTransaction).not.toHaveBeenCalled();
|
||||
expect(conn.commit).not.toHaveBeenCalled();
|
||||
expect(conn.release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not roll back a failed readOnly batch', async () => {
|
||||
const conn = makeConnection((sql: string) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ER_LOCK_DEADLOCK');
|
||||
return [[{ n: 0 }], undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
readOnly: true,
|
||||
});
|
||||
|
||||
await Promise.all([
|
||||
batcher.query('SELECT a', []),
|
||||
batcher.query('SELECT b', []),
|
||||
]);
|
||||
expect(conn.rollback).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('drops the oldest item with reason queueOverflow at the high-water mark', async () => {
|
||||
const conn = makeConnection(() => [[{ n: 0 }], undefined]);
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
maxQueueSize: 1,
|
||||
});
|
||||
|
||||
const first = batcher.query('SELECT a', []);
|
||||
const second = batcher.query('SELECT b', []);
|
||||
|
||||
await expect(first).rejects.toMatchObject({
|
||||
code: 'dbBatchFailed',
|
||||
reason: 'queueOverflow',
|
||||
});
|
||||
await expect(second).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it('rejects with reason connAcquire after exhausting acquisition retries', async () => {
|
||||
const getConnection = vi.fn(async () => {
|
||||
throw makeError('ECONNREFUSED');
|
||||
});
|
||||
const pool = { promise: () => ({ getConnection }) };
|
||||
const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 });
|
||||
|
||||
await expect(batcher.query('SELECT a', [])).rejects.toMatchObject({
|
||||
code: 'dbBatchFailed',
|
||||
reason: 'connAcquire',
|
||||
});
|
||||
expect(getConnection).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('bounds connection acquisition and rejects when the pool never answers', async () => {
|
||||
const getConnection = vi.fn(
|
||||
() => new Promise(() => {}), // pool never yields a connection
|
||||
);
|
||||
const pool = { promise: () => ({ getConnection }) };
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
acquireTimeoutMs: 30,
|
||||
});
|
||||
|
||||
await expect(batcher.query('SELECT a', [])).rejects.toMatchObject({
|
||||
code: 'dbBatchFailed',
|
||||
reason: 'connAcquire',
|
||||
});
|
||||
expect(getConnection).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('opens the breaker after consecutive failures and rejects with reason breakerOpen', async () => {
|
||||
// Batch and fallback both fail with an ambiguous (non-retriable for
|
||||
// writes) connection error, so no item gets through.
|
||||
const conn = makeConnection(() => {
|
||||
throw makeError('ECONNRESET');
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
failureThreshold: 1,
|
||||
cooldownMs: 60_000,
|
||||
});
|
||||
|
||||
await expect(batcher.query('INSERT x', [])).rejects.toMatchObject({
|
||||
code: 'ECONNRESET',
|
||||
});
|
||||
await expect(batcher.query('INSERT y', [])).rejects.toMatchObject({
|
||||
code: 'dbBatchFailed',
|
||||
reason: 'breakerOpen',
|
||||
});
|
||||
});
|
||||
|
||||
it('retries transient fallback failures when readOnly', async () => {
|
||||
let fallbackAttempts = 0;
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ECONNRESET');
|
||||
fallbackAttempts++;
|
||||
if (fallbackAttempts === 1) throw makeError('ECONNRESET');
|
||||
return [[{ ok: 1 }], undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
readOnly: true,
|
||||
});
|
||||
|
||||
const result = await batcher.query('SELECT a', []);
|
||||
expect(result[0]).toEqual([{ ok: 1 }]);
|
||||
expect(fallbackAttempts).toBe(2);
|
||||
});
|
||||
|
||||
it('does not retry ambiguous failures on a batcher that carries writes', async () => {
|
||||
let fallbackAttempts = 0;
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ECONNRESET');
|
||||
fallbackAttempts++;
|
||||
throw makeError('ECONNRESET');
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 });
|
||||
|
||||
await expect(batcher.query('INSERT x', [])).rejects.toMatchObject({
|
||||
code: 'ECONNRESET',
|
||||
});
|
||||
expect(fallbackAttempts).toBe(1);
|
||||
});
|
||||
|
||||
it('retries never-sent failures even on a batcher that carries writes', async () => {
|
||||
let fallbackAttempts = 0;
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ECONNRESET');
|
||||
fallbackAttempts++;
|
||||
if (fallbackAttempts === 1) throw makeError('ECONNREFUSED');
|
||||
return [[{ ok: 1 }], undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 });
|
||||
|
||||
const result = await batcher.query('INSERT x', []);
|
||||
expect(result[0]).toEqual([{ ok: 1 }]);
|
||||
expect(fallbackAttempts).toBe(2);
|
||||
});
|
||||
|
||||
it('retries a deadlocked write instead of surfacing it', async () => {
|
||||
let fallbackAttempts = 0;
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ER_LOCK_DEADLOCK');
|
||||
fallbackAttempts++;
|
||||
if (fallbackAttempts === 1) throw makeError('ER_LOCK_DEADLOCK');
|
||||
return [[{ ok: 1 }], undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 });
|
||||
|
||||
// The victim statement was rolled back by the server, so re-running it
|
||||
// can't double-apply — the caller should never see the deadlock.
|
||||
const result = await batcher.query('UPDATE notification SET x', []);
|
||||
expect(result[0]).toEqual([{ ok: 1 }]);
|
||||
expect(fallbackAttempts).toBe(2);
|
||||
});
|
||||
|
||||
it('gives up on a write that deadlocks past the retry budget', async () => {
|
||||
const conn = makeConnection(() => {
|
||||
throw makeError('ER_LOCK_DEADLOCK');
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 });
|
||||
|
||||
await expect(batcher.query('UPDATE hot SET x', [])).rejects.toMatchObject(
|
||||
{ code: 'ER_LOCK_DEADLOCK' },
|
||||
);
|
||||
});
|
||||
|
||||
it('sends a lone write as-is, outside a transaction', async () => {
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw new Error('unexpected batch query');
|
||||
return [{ affectedRows: 1 }, undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxBatchSize: 1 });
|
||||
|
||||
const result = await batcher.query('UPDATE t SET x = ?', [1]);
|
||||
expect(result[0]).toEqual({ affectedRows: 1 });
|
||||
expect(conn.query).toHaveBeenCalledTimes(1);
|
||||
expect(conn.query).toHaveBeenCalledWith('UPDATE t SET x = ?', [1]);
|
||||
expect(conn.beginTransaction).not.toHaveBeenCalled();
|
||||
expect(conn.commit).not.toHaveBeenCalled();
|
||||
expect(conn.release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('runs a lone write the server rejects once, without tripping the breaker', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const conn = makeConnection((sql) => {
|
||||
if (sql === 'INSERT dup') throw makeError('ER_DUP_ENTRY');
|
||||
return [{ affectedRows: 1 }, undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxBatchSize: 1,
|
||||
failureThreshold: 1,
|
||||
cooldownMs: 60_000,
|
||||
});
|
||||
|
||||
await expect(batcher.query('INSERT dup', [])).rejects.toMatchObject({
|
||||
code: 'ER_DUP_ENTRY',
|
||||
});
|
||||
expect(conn.query).toHaveBeenCalledTimes(1);
|
||||
expect(conn.rollback).not.toHaveBeenCalled();
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
// The server answered, so it is up: the next write goes through.
|
||||
await expect(batcher.query('INSERT fine', [])).resolves.toBeTruthy();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('retries a lone write on a connection the server closed for inactivity', async () => {
|
||||
let attempts = 0;
|
||||
const conn = makeConnection(() => {
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
throw makeError('ER_CLIENT_INTERACTION_TIMEOUT');
|
||||
}
|
||||
return [{ affectedRows: 1 }, undefined];
|
||||
});
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxBatchSize: 1 });
|
||||
|
||||
const result = await batcher.query('INSERT x', []);
|
||||
expect(result[0]).toEqual({ affectedRows: 1 });
|
||||
expect(attempts).toBe(2);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('never retries deterministic row-level errors and does not escalate the breaker', async () => {
|
||||
let fallbackAttempts = 0;
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ER_DUP_ENTRY');
|
||||
fallbackAttempts++;
|
||||
if (sql === 'INSERT dup') throw makeError('ER_DUP_ENTRY');
|
||||
return [[{ ok: 1 }], undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
failureThreshold: 1,
|
||||
cooldownMs: 60_000,
|
||||
readOnly: true,
|
||||
});
|
||||
|
||||
const dup = batcher.query('INSERT dup', []);
|
||||
const fine = batcher.query('INSERT fine', []);
|
||||
await expect(dup).rejects.toMatchObject({ code: 'ER_DUP_ENTRY' });
|
||||
await expect(fine).resolves.toBeTruthy();
|
||||
expect(fallbackAttempts).toBe(2);
|
||||
|
||||
// One fallback item succeeded, so the breaker must stay closed.
|
||||
const conn2 = makeConnection(happyBatch);
|
||||
// reuse same batcher/pool: next query must not be rejected upfront
|
||||
conn.query.mockImplementation(conn2.query.getMockImplementation()!);
|
||||
await expect(batcher.query('SELECT a', [])).resolves.toBeTruthy();
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
/*
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
@@ -21,20 +21,14 @@ import { existsSync, mkdirSync, readFileSync } from 'fs';
|
||||
import { basename, dirname, extname, join, resolve } from 'path';
|
||||
import { createContext, runInContext } from 'vm';
|
||||
import type { IConfig } from '../../types';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
import { Span } from '../../util/span.js';
|
||||
import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
|
||||
|
||||
const MIGRATIONS_DIR = resolve(__dirname, './migrations/sqlite');
|
||||
const STATEMENT_CACHE_SIZE = 1000;
|
||||
|
||||
type SqliteDatabase = InstanceType<typeof import('better-sqlite3')>;
|
||||
type SqliteStatement = import('better-sqlite3').Statement<unknown[]>;
|
||||
|
||||
/**
|
||||
* Ordered list of [threshold_version, files[]] pairs. A database whose
|
||||
* `user_version` is <= threshold_version will have the corresponding files
|
||||
* applied.
|
||||
* Ordered list of [threshold_version, files[]] pairs.
|
||||
* A database whose `user_version` is <= threshold_version will have
|
||||
* the corresponding files applied.
|
||||
*/
|
||||
const AVAILABLE_MIGRATIONS: [number, string[]][] = [
|
||||
[-1, ['0001_create-tables.sql', '0002_add-default-apps.sql']],
|
||||
@@ -84,73 +78,13 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
|
||||
[43, ['0047_app-url-updates.sql']],
|
||||
[44, ['0048_old-app-names-unique-tuple.sql']],
|
||||
[45, ['0049_music-player-pdf-player-updates.sql']],
|
||||
[46, ['0050_add_preamble_version.sql']],
|
||||
[47, ['0051_sessions_v2.sql']],
|
||||
[48, ['0052_sessions_v2_lookups.sql']],
|
||||
[49, ['0053_sessions_access_token_uid.sql']],
|
||||
[50, ['0054_sessions_workers.sql']],
|
||||
[50, ['0055_username_nocase_unique.sql']],
|
||||
[51, ['0056_sessions_kind_worker.sql']],
|
||||
[52, ['0057_add_user_reputation.sql']],
|
||||
[53, ['0058_add_phone_verification.sql']],
|
||||
[54, ['0059_add_card_verification.sql']],
|
||||
[55, ['0060_add_card_fingerprint.sql']],
|
||||
[56, ['0061_add_suspended_at.sql']],
|
||||
[57, ['0062_blocked-app-origins.sql']],
|
||||
[58, ['0063_add_suspended_reason.sql']],
|
||||
[59, ['0064_abuse-moderation-events.sql']],
|
||||
[60, ['0065_app-feedback.sql']],
|
||||
[61, ['0066_owned-email-unique.sql']],
|
||||
[62, ['0067_share_entries.sql']],
|
||||
[63, ['0068_referral-code-unique.sql']],
|
||||
[64, ['0069_user-block.sql']],
|
||||
[65, ['0070_drop-orphaned-default-groups.sql']],
|
||||
[66, ['0071_share_issuer_index.sql']],
|
||||
[67, ['0072_notification-scope.sql']],
|
||||
[68, ['0073_notification-created-at.sql']],
|
||||
[69, ['0074_add_user_home.sql']],
|
||||
[70, ['0075_event-subscriptions.sql']],
|
||||
[71, ['0076_event-handlers.sql']],
|
||||
[72, ['0077_teams.sql']],
|
||||
[73, ['0078_jct-user-group-pair-unique.sql']],
|
||||
[74, ['0079_team-audit-and-group-shares.sql']],
|
||||
[75, ['0080_kv-share-handles.sql']],
|
||||
[76, ['0081_event-subscriptions-indexes.sql']],
|
||||
[77, ['0082_temp-password-expiry.sql']],
|
||||
[78, ['0083_team-directory.sql']],
|
||||
[79, ['0084_share-anyone-with-link.sql']],
|
||||
[80, ['0085_event-subscriptions-include-value.sql']],
|
||||
[81, ['0086_subdomains-app-owner-cascade.sql']],
|
||||
[82, ['0087_feedback-attachments.sql']],
|
||||
[83, ['0088_apps-index-url.sql']],
|
||||
[84, ['0089_team-require-2fa.sql']],
|
||||
[85, ['0090_user-app-permission-index.sql']],
|
||||
[86, ['0091_icloud-dot-significance.sql']],
|
||||
[87, ['0092_share-entry-owner.sql']],
|
||||
[88, ['0093_fsentries-parent-uid-index.sql']],
|
||||
[89, ['0094_deleted-apps.sql']],
|
||||
[90, ['0095_kv-share-handle-levels.sql']],
|
||||
[91, ['0096_kv-share-handle-quota-indexes.sql']],
|
||||
[92, ['0097_user-to-user-holder-permission-index.sql']],
|
||||
[93, ['0098_notification-groups.sql']],
|
||||
[94, ['0099_kv-share-handle-notify.sql']],
|
||||
[95, ['0100_notification-send-key.sql']],
|
||||
[96, ['0101_user-scoped-handlers.sql']],
|
||||
[97, ['0102_notification-scope-id-index.sql']],
|
||||
[98, ['0103_share-entry-owner-backfill.sql']],
|
||||
[99, ['0104_signup-tls-fingerprint.sql']],
|
||||
[100, ['0105_sessions_kind_debug_app.sql']],
|
||||
];
|
||||
|
||||
export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
override readonly engineName = 'sqlite';
|
||||
|
||||
// better-sqlite3 instance — set during onServerStart
|
||||
private db!: SqliteDatabase;
|
||||
/** Prepared statements by query text; SQLite re-prepares on schema change. */
|
||||
private readonly statements = new BoundedTtlMap<string, SqliteStatement>({
|
||||
maxEntries: STATEMENT_CACHE_SIZE,
|
||||
});
|
||||
private db!: InstanceType<typeof import('better-sqlite3')>;
|
||||
|
||||
constructor(config: IConfig) {
|
||||
super(config);
|
||||
@@ -172,14 +106,12 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
mkdirSync(dirname(dbPath), { recursive: true });
|
||||
}
|
||||
|
||||
this.statements.clear();
|
||||
this.db = new Database(dbPath);
|
||||
|
||||
await this.runMigrations(isNew);
|
||||
}
|
||||
|
||||
override onServerShutdown(): void {
|
||||
this.statements.clear();
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
@@ -189,14 +121,16 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
// Query interface
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
@Span('db.read', (query: string) => ({ 'db.statement': query }))
|
||||
override async read(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
return this.prepare(query).all(
|
||||
...this.transformParams(params),
|
||||
) as Record<string, unknown>[];
|
||||
query = this.transformQuery(query);
|
||||
params = this.transformParams(params);
|
||||
return this.db.prepare(query).all(...params) as Record<
|
||||
string,
|
||||
unknown
|
||||
>[];
|
||||
}
|
||||
|
||||
override async pread(
|
||||
@@ -207,32 +141,30 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
@Span('db.write', (query: string) => ({ 'db.statement': query }))
|
||||
override async write(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<WriteResult> {
|
||||
const info = this.prepare(query).run(...this.transformParams(params));
|
||||
query = this.transformQuery(query);
|
||||
params = this.transformParams(params);
|
||||
|
||||
const info = this.db.prepare(query).run(...params);
|
||||
|
||||
return {
|
||||
insertId: info.lastInsertRowid,
|
||||
affectedRows: info.changes,
|
||||
anyRowsAffected: info.changes > 0,
|
||||
// No warningStatus: SQLite raises none, but a new row can reuse a
|
||||
// deleted row's rowid that other tables still name, so a clean
|
||||
// insert doesn't prove the row is all the caller sent.
|
||||
};
|
||||
}
|
||||
|
||||
@Span('db.batchWrite', (entries: unknown[]) => ({
|
||||
'db.batch_size': entries.length,
|
||||
}))
|
||||
override async batchWrite(
|
||||
entries: { statement: string; values: unknown[] }[],
|
||||
): Promise<void> {
|
||||
this.db.transaction(() => {
|
||||
for (const { statement, values } of entries) {
|
||||
this.prepare(statement).run(...this.transformParams(values));
|
||||
for (let { statement, values } of entries) {
|
||||
statement = this.transformQuery(statement);
|
||||
values = this.transformParams(values);
|
||||
this.db.prepare(statement).run(...values);
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -241,15 +173,6 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
// SQLite-specific transforms
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private prepare(query: string): SqliteStatement {
|
||||
let statement = this.statements.get(query);
|
||||
if (!statement) {
|
||||
statement = this.db.prepare(this.transformQuery(query));
|
||||
this.statements.set(query, statement);
|
||||
}
|
||||
return statement;
|
||||
}
|
||||
|
||||
private transformQuery(query: string): string {
|
||||
return query.replace(/now\(\)/gi, "datetime('now')");
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/*
|
||||
/**
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
@@ -24,18 +24,16 @@ export {
|
||||
} from './DatabaseClient';
|
||||
export { SqliteDatabaseClient } from './SqliteDatabaseClient';
|
||||
export { MySQLDatabaseClient } from './MySQLDatabaseClient';
|
||||
export { PostgresDatabaseClient } from './PostgresDatabaseClient';
|
||||
|
||||
import type { IConfig } from '../../types';
|
||||
import { AbstractDatabaseClient } from './DatabaseClient';
|
||||
import { MySQLDatabaseClient } from './MySQLDatabaseClient';
|
||||
import { PostgresDatabaseClient } from './PostgresDatabaseClient';
|
||||
import { SqliteDatabaseClient } from './SqliteDatabaseClient';
|
||||
|
||||
/**
|
||||
* Factory class registered in `puterClients`. PuterServer calls `new
|
||||
* DatabaseClientFactory(config)` — the constructor returns the concrete
|
||||
* subclass selected by `config.database.engine`.
|
||||
* Factory class registered in `puterClients`. PuterServer calls
|
||||
* `new DatabaseClientFactory(config)` — the constructor returns the
|
||||
* concrete subclass selected by `config.database.engine`.
|
||||
*/
|
||||
export const DatabaseClientFactory = class DatabaseClientFactory {
|
||||
constructor(config: IConfig) {
|
||||
@@ -43,8 +41,6 @@ export const DatabaseClientFactory = class DatabaseClientFactory {
|
||||
switch (engine) {
|
||||
case 'mysql':
|
||||
return new MySQLDatabaseClient(config);
|
||||
case 'postgres':
|
||||
return new PostgresDatabaseClient(config);
|
||||
case 'sqlite':
|
||||
return new SqliteDatabaseClient(config);
|
||||
default:
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Comparator for migration filenames named `<engine>_mig_<N>.sql`.
|
||||
*
|
||||
* Existing MySQL files use unpadded integers, so lexical sorting places
|
||||
* `*_mig_10.sql` before `*_mig_2.sql`. Pull the trailing integer out and sort
|
||||
* numerically. Anything that does not match the `_<digits>.sql` shape falls
|
||||
* back to lexical comparison and sorts after numbered files.
|
||||
*/
|
||||
export const compareMigrationFilenames = (a: string, b: string): number => {
|
||||
const numericIndex = (name: string): number => {
|
||||
const m = /_(\d+)\.sql$/.exec(name);
|
||||
return m ? Number.parseInt(m[1], 10) : Number.NaN;
|
||||
};
|
||||
const na = numericIndex(a);
|
||||
const nb = numericIndex(b);
|
||||
if (Number.isFinite(na) && Number.isFinite(nb)) {
|
||||
if (na !== nb) return na - nb;
|
||||
} else if (Number.isFinite(na)) {
|
||||
return -1;
|
||||
} else if (Number.isFinite(nb)) {
|
||||
return 1;
|
||||
}
|
||||
return a.localeCompare(b);
|
||||
};
|
||||
@@ -24,9 +24,7 @@
|
||||
--
|
||||
|
||||
--
|
||||
-- Idempotent column-ensure helper. Used by the CALLs below AND by later
|
||||
-- migration files, which CALL it as a one-line idempotent column add. Left
|
||||
-- resident (not dropped at end of file) so it outlives this migration.
|
||||
-- Idempotent column-ensure helper (used by CALLs below; dropped at end of file)
|
||||
--
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_col;
|
||||
@@ -1060,7 +1058,6 @@ CREATE TABLE IF NOT EXISTS `user` (
|
||||
`signup_server` varchar(255) DEFAULT NULL,
|
||||
`metadata` json DEFAULT (json_object()),
|
||||
`reputation` smallint DEFAULT '100',
|
||||
`home` varchar(64) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uid` (`uuid`),
|
||||
UNIQUE KEY `username` (`username`),
|
||||
@@ -1152,7 +1149,6 @@ CALL _puter_add_col('user', 'signup_user_agent', '`signup_user_agent` varchar(51
|
||||
CALL _puter_add_col('user', 'signup_origin', '`signup_origin` varchar(255) DEFAULT NULL');
|
||||
CALL _puter_add_col('user', 'signup_server', '`signup_server` varchar(255) DEFAULT NULL');
|
||||
CALL _puter_add_col('user', 'metadata', '`metadata` json DEFAULT (json_object())');
|
||||
CALL _puter_add_col('user', 'home', '`home` varchar(64) DEFAULT NULL');
|
||||
CALL _puter_add_col('user', 'reputation', '`reputation` smallint DEFAULT ''100''');
|
||||
|
||||
--
|
||||
@@ -1337,10 +1333,7 @@ CALL _puter_add_col('user_update_audit', 'new_username', '`new_username` varchar
|
||||
CALL _puter_add_col('user_update_audit', 'reason', '`reason` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL');
|
||||
|
||||
|
||||
-- _puter_add_col is intentionally NOT dropped here: later migration files CALL
|
||||
-- it as a one-line idempotent column add, and migrations replay on every boot
|
||||
-- with no applied-state tracking, so the helper must outlive this file. The
|
||||
-- DROP-before-CREATE at the top keeps this migration itself replay-safe.
|
||||
DROP PROCEDURE IF EXISTS _puter_add_col;
|
||||
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
|
||||
|
||||
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0053. Adds the
|
||||
-- `access_token_uid` reverse-lookup column on `sessions` so raw-uuid
|
||||
-- revoke can find the matching session row when only the v2 token_uid
|
||||
-- (no JWT) is presented.
|
||||
--
|
||||
-- Idempotent: each ADD COLUMN / ADD INDEX is guarded so the migration
|
||||
-- directory can be replayed safely.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_sessions_access_token_uid;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_sessions_access_token_uid()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'sessions' AND COLUMN_NAME = 'access_token_uid'
|
||||
) THEN
|
||||
ALTER TABLE `sessions`
|
||||
ADD COLUMN `access_token_uid` VARCHAR(64) DEFAULT NULL;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'sessions'
|
||||
AND INDEX_NAME = 'idx_sessions_access_token_uid'
|
||||
) THEN
|
||||
ALTER TABLE `sessions`
|
||||
ADD INDEX `idx_sessions_access_token_uid` (`access_token_uid`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_sessions_access_token_uid();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_sessions_access_token_uid;
|
||||
@@ -1,70 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Worker session uniqueness. Each Puter worker is a separately-deployed
|
||||
-- code unit (its own subdomain), so one user can have many workers under
|
||||
-- the same app — distinguished by `meta.worker_name`. The natural unique
|
||||
-- key for an active worker session is therefore
|
||||
-- (user_id, app_uid, worker_name), and app_uid is allowed NULL for
|
||||
-- user-scoped workers that aren't bound to any specific app.
|
||||
--
|
||||
-- Implemented the same way as `app_unique_key` from mig_9: a VIRTUAL
|
||||
-- generated column that's non-NULL only for the rows under the rule,
|
||||
-- then a UNIQUE INDEX on the column. NULLs don't conflict in MySQL
|
||||
-- UNIQUE indexes, so soft-revoked / non-worker rows fall out
|
||||
-- automatically. IFNULL normalises NULL `app_uid` so two user-scoped
|
||||
-- workers with the same name still dedupe. JSON_UNQUOTE strips the
|
||||
-- JSON value quoting from JSON_EXTRACT so the concatenated key is a
|
||||
-- plain string that matches the SELECT path's binding.
|
||||
--
|
||||
-- Idempotent: each ADD COLUMN / ADD INDEX is guarded so the migration
|
||||
-- directory can be replayed safely.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_sessions_worker_unique;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_sessions_worker_unique()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'sessions' AND COLUMN_NAME = 'worker_unique_key'
|
||||
) THEN
|
||||
ALTER TABLE `sessions`
|
||||
ADD COLUMN `worker_unique_key` VARCHAR(550)
|
||||
GENERATED ALWAYS AS (
|
||||
IF(`kind` = 'worker' AND `revoked_at` IS NULL,
|
||||
CONCAT(`user_id`, '|', IFNULL(`app_uid`, ''), '|',
|
||||
IFNULL(JSON_UNQUOTE(JSON_EXTRACT(`meta`, '$.worker_name')), '')),
|
||||
NULL)
|
||||
) VIRTUAL;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'sessions'
|
||||
AND INDEX_NAME = 'idx_sessions_user_worker_active'
|
||||
) THEN
|
||||
ALTER TABLE `sessions`
|
||||
ADD UNIQUE INDEX `idx_sessions_user_worker_active` (`worker_unique_key`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_sessions_worker_unique();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_sessions_worker_unique;
|
||||
@@ -1,50 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Add 'worker' to the `sessions.kind` ENUM. mig_11 already added the
|
||||
-- worker uniqueness index and the application code in SessionStore
|
||||
-- inserts rows with kind='worker', but the ENUM defined in mig_8 never
|
||||
-- listed 'worker' as a permitted value. Under STRICT_TRANS_TABLES the
|
||||
-- INSERT fails outright; under a relaxed sql_mode the value is coerced
|
||||
-- to '' (and the worker SELECT-by-kind path then misses the row).
|
||||
-- Mirrors SQLite migration 0056.
|
||||
--
|
||||
-- Idempotent: guarded against COLUMN_TYPE so re-running the directory is
|
||||
-- a no-op once 'worker' is in the ENUM.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_sessions_kind_worker;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_sessions_kind_worker()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'sessions'
|
||||
AND COLUMN_NAME = 'kind'
|
||||
AND FIND_IN_SET('worker', REPLACE(REPLACE(REPLACE(COLUMN_TYPE, 'enum(', ''), ')', ''), '''', '')) > 0
|
||||
) THEN
|
||||
ALTER TABLE `sessions`
|
||||
MODIFY COLUMN `kind`
|
||||
ENUM('web', 'app', 'access_token', 'asset', 'worker')
|
||||
NOT NULL DEFAULT 'web';
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_sessions_kind_worker();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_sessions_kind_worker;
|
||||
@@ -1,47 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- SMS phone verification columns. Mirrors SQLite migration 0058. `phone` is the
|
||||
-- E.164 number collected during verification (indexed like `email`);
|
||||
-- `requires_phone_verification` gates account use for low-reputation signups
|
||||
-- (not indexed, mirroring `requires_email_confirmation`).
|
||||
--
|
||||
-- Idempotent: column adds use _puter_add_col (defined in mig_1, which leaves it
|
||||
-- resident for later migrations); the index add is guarded against
|
||||
-- INFORMATION_SCHEMA.STATISTICS so the directory replays safely.
|
||||
|
||||
CALL _puter_add_col('user', 'phone', '`phone` varchar(20) DEFAULT NULL');
|
||||
CALL _puter_add_col('user', 'requires_phone_verification', '`requires_phone_verification` tinyint(1) NOT NULL DEFAULT ''0''');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_phone_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_user_phone_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user'
|
||||
AND INDEX_NAME = 'idx_user_phone'
|
||||
) THEN
|
||||
ALTER TABLE `user` ADD INDEX `idx_user_phone` (`phone`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_user_phone_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_phone_index;
|
||||
@@ -1,26 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Credit-card verification column. Mirrors SQLite migration 0059.
|
||||
-- `requires_card_verification` gates account use for low-reputation signups;
|
||||
-- the card itself never touches our DB, so this is the only column (not
|
||||
-- indexed, mirroring `requires_phone_verification`).
|
||||
--
|
||||
-- Idempotent: the column add uses _puter_add_col (from mig_1) so the
|
||||
-- directory replays safely.
|
||||
|
||||
CALL _puter_add_col('user', 'requires_card_verification', '`requires_card_verification` tinyint(1) NOT NULL DEFAULT ''0''');
|
||||
@@ -1,47 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Card fingerprint column. Mirrors SQLite migration 0060. `card_fingerprint`
|
||||
-- is the Stripe card fingerprint (stable per card number) recorded when a user
|
||||
-- clears card verification — the card sibling of `phone`, indexed like it so
|
||||
-- admin tooling can find the accounts that verified with a given card. The card
|
||||
-- itself never touches our DB, only Stripe's fingerprint for it.
|
||||
--
|
||||
-- Idempotent: the column add uses _puter_add_col (defined in mig_1, which
|
||||
-- leaves it resident for later migrations); the index add is guarded against
|
||||
-- INFORMATION_SCHEMA.STATISTICS so the directory replays safely.
|
||||
|
||||
CALL _puter_add_col('user', 'card_fingerprint', '`card_fingerprint` varchar(128) DEFAULT NULL');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_card_fingerprint_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_user_card_fingerprint_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user'
|
||||
AND INDEX_NAME = 'idx_user_card_fingerprint'
|
||||
) THEN
|
||||
ALTER TABLE `user` ADD INDEX `idx_user_card_fingerprint` (`card_fingerprint`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_user_card_fingerprint_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_card_fingerprint_index;
|
||||
@@ -1,46 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Suspended-at column. Mirrors SQLite migration 0061. `suspended_at` is when
|
||||
-- the account was suspended, as unix seconds (NULL while not suspended) — the
|
||||
-- timestamp sibling of the boolean `suspended` flag, indexed so the
|
||||
-- signup-abuse harness can count an IP's recently-suspended accounts.
|
||||
--
|
||||
-- Idempotent: the column add uses _puter_add_col (defined in mig_1, which
|
||||
-- leaves it resident for later migrations); the index add is guarded against
|
||||
-- INFORMATION_SCHEMA.STATISTICS so the directory replays safely.
|
||||
|
||||
CALL _puter_add_col('user', 'suspended_at', '`suspended_at` bigint DEFAULT NULL');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_suspended_at_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_user_suspended_at_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user'
|
||||
AND INDEX_NAME = 'idx_user_suspended_at'
|
||||
) THEN
|
||||
ALTER TABLE `user` ADD INDEX `idx_user_suspended_at` (`suspended_at`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_user_suspended_at_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_suspended_at_index;
|
||||
@@ -1,36 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Admin-managed blocklist of app origins. Mirrors SQLite migration 0062.
|
||||
-- An app whose `index_url` host (or a request origin) matches an entry is
|
||||
-- denied access to Puter resources: it cannot obtain an app token and
|
||||
-- already-issued app tokens are rejected on each request. `include_subdomains
|
||||
-- = 1` also blocks every subdomain of `domain`. Enforced in AuthService via
|
||||
-- AppOriginBlocklistService.
|
||||
--
|
||||
-- Idempotent: `CREATE TABLE IF NOT EXISTS` lets the directory replay safely.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `blocked_app_origins` (
|
||||
`id` INT NOT NULL AUTO_INCREMENT,
|
||||
`domain` VARCHAR(255) NOT NULL,
|
||||
`include_subdomains` TINYINT(1) NOT NULL DEFAULT 0,
|
||||
`reason` TEXT DEFAULT NULL,
|
||||
`created_by` VARCHAR(255) DEFAULT NULL,
|
||||
`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `idx_blocked_app_origins_domain` (`domain`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -1,26 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Suspension reason column. Mirrors SQLite migration 0063. Why an account was
|
||||
-- suspended (NULL while not suspended) — companion to the boolean `suspended`
|
||||
-- flag and `suspended_at` timestamp. Constrained at the application layer to a
|
||||
-- fixed set of reasons (see extensions/admin suspension_reasons.js).
|
||||
--
|
||||
-- Idempotent: the column add uses _puter_add_col (defined in mig_1, which
|
||||
-- leaves it resident for later migrations).
|
||||
|
||||
CALL _puter_add_col('user', 'suspended_reason', '`suspended_reason` VARCHAR(64) DEFAULT NULL');
|
||||
@@ -1,38 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Append-only log of admin moderation actions. Mirrors SQLite migration 0064.
|
||||
-- Used to measure the abuse system's false-positive rate: `unsuspend` (admin
|
||||
-- unblock) and `admin_create_user` are the false-positive signals; `suspend`
|
||||
-- gives the denominator. Preserves history the `user` suspension columns lose
|
||||
-- on unsuspend. `created_at` is unix seconds (matching `user.suspended_at`).
|
||||
--
|
||||
-- Idempotent: `CREATE TABLE IF NOT EXISTS` lets the directory replay safely.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `abuse_moderation_events` (
|
||||
`id` INT NOT NULL AUTO_INCREMENT,
|
||||
`action` VARCHAR(32) NOT NULL,
|
||||
`target_user_id` BIGINT DEFAULT NULL,
|
||||
`target_username` VARCHAR(255) DEFAULT NULL,
|
||||
`admin_username` VARCHAR(255) DEFAULT NULL,
|
||||
`reason` TEXT DEFAULT NULL,
|
||||
`source` VARCHAR(64) DEFAULT NULL,
|
||||
`created_at` BIGINT NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_abuse_moderation_events_created_at` (`created_at`),
|
||||
KEY `idx_abuse_moderation_events_action` (`action`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -1,48 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- User-to-developer feedback for apps that opt in. Mirrors SQLite migration
|
||||
-- 0065. Opt-in is the new `apps.feedback_enabled` column (developer-writable
|
||||
-- through the regular `puter.apps.update` path). Each row of `app_feedback`
|
||||
-- is one message a signed-in user submitted through the GUI feedback dialog;
|
||||
-- a copy is emailed to the app owner unless the per-app daily email cap
|
||||
-- suppressed it (`email_sent` records which). `app_uid` is denormalized
|
||||
-- alongside `app_id` so rows stay attributable after an app is deleted.
|
||||
-- `created_at` is unix seconds.
|
||||
--
|
||||
-- Idempotent: the column add uses _puter_add_col (defined in mig_1, which
|
||||
-- always runs first) and the table uses `CREATE TABLE IF NOT EXISTS`, so the
|
||||
-- directory can replay safely.
|
||||
|
||||
CALL _puter_add_col('apps', 'feedback_enabled', '`feedback_enabled` tinyint(1) DEFAULT ''0''');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `app_feedback` (
|
||||
`id` INT NOT NULL AUTO_INCREMENT,
|
||||
`uid` CHAR(36) NOT NULL,
|
||||
`app_id` INT NOT NULL,
|
||||
`app_uid` CHAR(40) NOT NULL,
|
||||
`user_id` INT NOT NULL,
|
||||
`message` TEXT NOT NULL,
|
||||
`source_env` VARCHAR(16) DEFAULT NULL,
|
||||
`source_origin` VARCHAR(2048) DEFAULT NULL,
|
||||
`email_sent` TINYINT(1) NOT NULL DEFAULT 0,
|
||||
`created_at` BIGINT NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uq_app_feedback_uid` (`uid`),
|
||||
KEY `idx_app_feedback_app_created` (`app_id`, `created_at`),
|
||||
KEY `idx_app_feedback_user_created` (`user_id`, `created_at`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -1,128 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Enforce "at most one account owns an email address". Mirrors SQLite
|
||||
-- migration 0066.
|
||||
--
|
||||
-- `user.email` is deliberately not UNIQUE: several rows may legitimately hold
|
||||
-- the same address while unconfirmed (admin-provisioned placeholders, signups
|
||||
-- that were never confirmed, temp accounts on their way to becoming real). What
|
||||
-- must never happen is two rows both *owning* an address — owning meaning the
|
||||
-- row is confirmed, or holds a password and so can drive password recovery for
|
||||
-- that inbox.
|
||||
--
|
||||
-- Signup, save-account, change-email, OIDC and admin provisioning each check for
|
||||
-- an owner before writing, but a check and an insert are not one operation: two
|
||||
-- requests can both read "free" and both write. This index is what actually
|
||||
-- holds the invariant; the application checks just produce a nicer error most of
|
||||
-- the time.
|
||||
--
|
||||
-- SQLite expresses that with a partial index. MySQL has none, so the predicate
|
||||
-- lives in a generated column that evaluates to NULL for every row that does not
|
||||
-- own its address — and NULLs do not collide in an InnoDB unique index, which is
|
||||
-- exactly the "unlimited unconfirmed placeholders" behaviour we need.
|
||||
--
|
||||
-- The column is VIRTUAL, not STORED, on purpose: adding a stored generated
|
||||
-- column rebuilds the table, while a virtual one is a metadata-only change and
|
||||
-- the index that follows builds INPLACE. On a `user` table of any size with
|
||||
-- read replicas attached, that is the difference between a routine change and an
|
||||
-- outage. The ALGORITHM/LOCK clauses are spelled out so a server that cannot
|
||||
-- honour them refuses the statement instead of quietly copying the table.
|
||||
--
|
||||
-- Matching is on the canonical address so provider aliases
|
||||
-- (`[email protected]` vs `[email protected]`) collide. `clean_email` is
|
||||
-- written on every modern write path; the COALESCE covers rows old enough to
|
||||
-- predate the column. Run the `clean_email` backfill before this migration or
|
||||
-- alias collisions among those rows go unnoticed.
|
||||
--
|
||||
-- Idempotent: both steps are guarded on INFORMATION_SCHEMA so the directory
|
||||
-- replays safely.
|
||||
--
|
||||
-- If the index creation fails with ER_DUP_ENTRY, the DB already contains
|
||||
-- duplicate owners. Collapse them first (admin → One-off Jobs → Collapse
|
||||
-- Duplicate Emails) — there is no safe automatic merge of two accounts.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_owned_email;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_owned_email()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user'
|
||||
AND COLUMN_NAME = 'owned_email'
|
||||
) THEN
|
||||
ALTER TABLE `user`
|
||||
ADD COLUMN `owned_email` VARCHAR(256)
|
||||
CHARACTER SET latin1 COLLATE latin1_swedish_ci
|
||||
GENERATED ALWAYS AS (
|
||||
CASE
|
||||
WHEN `email` IS NOT NULL
|
||||
AND (`email_confirmed` = 1 OR `password` IS NOT NULL)
|
||||
THEN COALESCE(`clean_email`, LOWER(`email`))
|
||||
ELSE NULL
|
||||
END
|
||||
) VIRTUAL,
|
||||
ALGORITHM=INSTANT;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user'
|
||||
AND INDEX_NAME = 'idx_user_owned_email'
|
||||
) THEN
|
||||
ALTER TABLE `user`
|
||||
ADD UNIQUE KEY `idx_user_owned_email` (`owned_email`),
|
||||
ALGORITHM=INPLACE, LOCK=NONE;
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_owned_email();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_owned_email;
|
||||
|
||||
-- One Puter account per external identity. OIDCStore.link already assumes this
|
||||
-- constraint exists — it catches the unique violation to tell "re-linking the
|
||||
-- same account" apart from "this sub belongs to someone else" — but the table
|
||||
-- never actually had it, so two concurrent first-time logins could each create
|
||||
-- an account and each link the same sub. Subsequent logins then resolved to
|
||||
-- whichever row came back first.
|
||||
--
|
||||
-- Dedupe `user_oidc_providers` before applying this: keep the lowest `id` per
|
||||
-- (provider, provider_sub) and point it at the account the collapse job kept.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_oidc_sub_unique;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_oidc_sub_unique()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user_oidc_providers'
|
||||
AND INDEX_NAME = 'idx_user_oidc_provider_sub'
|
||||
) THEN
|
||||
ALTER TABLE `user_oidc_providers`
|
||||
ADD UNIQUE KEY `idx_user_oidc_provider_sub` (`provider`, `provider_sub`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_oidc_sub_unique();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_oidc_sub_unique;
|
||||
@@ -1,106 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Grow `share` from a pending-email-invite table into the index of active
|
||||
-- shares. See sqlite/0067_share_entries.sql for the column rationale.
|
||||
--
|
||||
-- Idempotent: columns go through _puter_add_col (from mig_1), indexes and
|
||||
-- foreign keys through the guarded procedure below. There is no per-file
|
||||
-- applied-state tracking, so every statement must tolerate a re-run.
|
||||
|
||||
CALL _puter_add_col('share', 'holder_user_id', '`holder_user_id` int unsigned DEFAULT NULL');
|
||||
CALL _puter_add_col('share', 'fsentry_id', '`fsentry_id` int unsigned DEFAULT NULL');
|
||||
CALL _puter_add_col('share', 'mode', '`mode` varchar(20) DEFAULT NULL');
|
||||
CALL _puter_add_col('share', 'applied_at', '`applied_at` timestamp NULL DEFAULT NULL');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_share_index_constraints;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_share_index_constraints()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND INDEX_NAME = 'idx_share_holder'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD INDEX `idx_share_holder` (`holder_user_id`, `id`);
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND INDEX_NAME = 'idx_share_fsentry'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD INDEX `idx_share_fsentry` (`fsentry_id`);
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND INDEX_NAME = 'idx_share_holder_entry_issuer'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD UNIQUE INDEX `idx_share_holder_entry_issuer`
|
||||
(`holder_user_id`, `fsentry_id`, `issuer_user_id`);
|
||||
END IF;
|
||||
|
||||
-- Retiring a deleted node's grants looks them up by permission text
|
||||
-- (`fs:<uuid>` and `fs:<uuid>:%`); the subject lives in that text rather
|
||||
-- than a column, so no foreign key can cascade it. The primary key is
|
||||
-- (issuer, holder, permission), which never puts `permission` first, so
|
||||
-- without this both the equality and the left-anchored LIKE degrade to a
|
||||
-- full scan on the delete path. The column is ascii varchar(255), well
|
||||
-- inside InnoDB's key limit, so it is indexed whole.
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user_to_user_permissions'
|
||||
AND INDEX_NAME = 'idx_user_to_user_permissions_permission'
|
||||
) THEN
|
||||
ALTER TABLE `user_to_user_permissions`
|
||||
ADD INDEX `idx_user_to_user_permissions_permission` (`permission`);
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND CONSTRAINT_NAME = 'share_holder_user_fk'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD CONSTRAINT `share_holder_user_fk`
|
||||
FOREIGN KEY (`holder_user_id`) REFERENCES `user` (`id`)
|
||||
ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
END IF;
|
||||
|
||||
-- The cascade that retires a share with its file.
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND CONSTRAINT_NAME = 'share_fsentry_fk'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD CONSTRAINT `share_fsentry_fk`
|
||||
FOREIGN KEY (`fsentry_id`) REFERENCES `fsentries` (`id`)
|
||||
ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_share_index_constraints();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_share_index_constraints;
|
||||
@@ -1,35 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- One user refusing contact from another. See sqlite/0068_user-block.sql for
|
||||
-- the rationale. `created_at` is unix seconds.
|
||||
--
|
||||
-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline,
|
||||
-- so the directory can replay safely.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `user_block` (
|
||||
`id` INT NOT NULL AUTO_INCREMENT,
|
||||
`blocker_user_id` INT UNSIGNED NOT NULL,
|
||||
`blocked_user_id` INT UNSIGNED NOT NULL,
|
||||
`created_at` BIGINT NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `idx_user_block_pair` (`blocker_user_id`, `blocked_user_id`),
|
||||
CONSTRAINT `fk_user_block_blocker` FOREIGN KEY (`blocker_user_id`)
|
||||
REFERENCES `user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
CONSTRAINT `fk_user_block_blocked` FOREIGN KEY (`blocked_user_id`)
|
||||
REFERENCES `user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -1,61 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Widen `notification.shown` and `notification.acknowledged` to hold the unix
|
||||
-- second the store has been writing to them since the backend rework.
|
||||
--
|
||||
-- Both arrived from the v1 schema as `tinyint(1)`, where they were flags. The
|
||||
-- rework changed the writes to timestamps and sqlite (`INTEGER`) and postgres
|
||||
-- (`bigint`) took them; only mysql was left too narrow, so every `markShown`
|
||||
-- and `markAcknowledged` there fails with ER_WARN_DATA_OUT_OF_RANGE and the
|
||||
-- column stays NULL. Dismissing a notification therefore never sticks, and one
|
||||
-- already delivered is re-sent on every reconnect.
|
||||
--
|
||||
-- No backfill: every reader tests `IS NULL` / `IS NOT NULL` only, so a legacy
|
||||
-- `1` keeps meaning "yes" once widened.
|
||||
--
|
||||
-- Guarded on the current type rather than run unconditionally. Changing a
|
||||
-- column type copies the table, and this one grows without bound — a migration
|
||||
-- directory that replays on every boot must not rebuild it every time.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_widen_notification_stamps;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_widen_notification_stamps()
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND COLUMN_NAME = 'shown'
|
||||
AND DATA_TYPE = 'tinyint'
|
||||
) THEN
|
||||
ALTER TABLE `notification` MODIFY `shown` BIGINT DEFAULT NULL;
|
||||
END IF;
|
||||
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND COLUMN_NAME = 'acknowledged'
|
||||
AND DATA_TYPE = 'tinyint'
|
||||
) THEN
|
||||
ALTER TABLE `notification` MODIFY `acknowledged` BIGINT DEFAULT NULL;
|
||||
END IF;
|
||||
END //
|
||||
DELIMITER ;
|
||||
CALL _puter_widen_notification_stamps();
|
||||
DROP PROCEDURE IF EXISTS _puter_widen_notification_stamps;
|
||||
@@ -1,22 +0,0 @@
|
||||
-- Drops three groups that no code has ever read: `freeai`, `experimental` and
|
||||
-- `dangerous`. No migration creates them — they were added by hand back when
|
||||
-- hardcoded permissions were keyed by group name. That map is now a flat
|
||||
-- per-user floor (`default_user_permissions` in data/hardcoded-permissions.js),
|
||||
-- so a group nothing looks up grants nothing, and the rows are dead weight.
|
||||
--
|
||||
-- Guarded, and deliberately so: `user_to_group_permissions.group_id` and
|
||||
-- `jct_user_group.group_id` both cascade on delete, so removing a group that
|
||||
-- still carries permissions or members would silently revoke them from every
|
||||
-- member. Only a group with neither is dropped. A group that survives this
|
||||
-- migration has dependents and needs a deliberate decision, not a sweep --
|
||||
-- query `user_to_group_permissions` by `group_id` to see what it holds.
|
||||
--
|
||||
-- The joins rather than `NOT IN` subqueries: MySQL will not read from the table
|
||||
-- it is deleting from inside a subquery on the same statement.
|
||||
DELETE g FROM `group` g
|
||||
LEFT JOIN `user_to_group_permissions` p ON p.`group_id` = g.`id`
|
||||
LEFT JOIN `jct_user_group` j ON j.`group_id` = g.`id`
|
||||
WHERE JSON_UNQUOTE(JSON_EXTRACT(g.`extra`, '$.name'))
|
||||
IN ('freeai', 'experimental', 'dangerous')
|
||||
AND p.`group_id` IS NULL
|
||||
AND j.`group_id` IS NULL;
|
||||
@@ -1,80 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Give `notification` a scope tuple: what the row is (`type`), the app it is
|
||||
-- about (`app_uid`, NULL for platform), and who the recipient is in relation
|
||||
-- to it (`audience`). Attribution lived inside the `value` JSON as two
|
||||
-- free-form markers, `source` and `template`, that could disagree; these
|
||||
-- columns replace them and are what a scoped list can be indexed on.
|
||||
--
|
||||
-- `app_uid` is char(40) to match `apps`.`uid`, which is `app-` plus a uuid.
|
||||
--
|
||||
-- Idempotent: columns go through `_puter_add_col` (mig_1), the index through
|
||||
-- the guarded procedure. There is no per-file applied-state tracking, so every
|
||||
-- statement here tolerates a re-run.
|
||||
|
||||
CALL _puter_add_col('notification', 'app_uid', '`app_uid` char(40) COLLATE utf8mb4_unicode_ci DEFAULT NULL');
|
||||
CALL _puter_add_col('notification', 'audience', '`audience` varchar(16) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''account''');
|
||||
CALL _puter_add_col('notification', 'type', '`type` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''''');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_scope_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_notification_scope_index()
|
||||
BEGIN
|
||||
-- Serves both the scoped list and the unacknowledged count.
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND INDEX_NAME = 'idx_notification_scope'
|
||||
) THEN
|
||||
ALTER TABLE `notification` ADD INDEX `idx_notification_scope`
|
||||
(`user_id`, `audience`, `app_uid`, `acknowledged`);
|
||||
END IF;
|
||||
END //
|
||||
DELIMITER ;
|
||||
CALL _puter_add_notification_scope_index();
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_scope_index;
|
||||
|
||||
-- Backfill from the markers the payload actually carried. `type = ''` is the
|
||||
-- not-yet-classified state, so re-running matches nothing the first pass
|
||||
-- claimed, and a row matching no marker keeps the defaults (`''` / 'account')
|
||||
-- and reads as legacy.
|
||||
|
||||
UPDATE `notification`
|
||||
SET `type` = 'share.received', `audience` = 'account', `app_uid` = NULL
|
||||
WHERE `type` = ''
|
||||
AND JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.template')) = 'file-shared-with-you';
|
||||
|
||||
UPDATE `notification`
|
||||
SET `type` = 'share.claimed', `audience` = 'account', `app_uid` = NULL
|
||||
WHERE `type` = ''
|
||||
AND JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.template')) = 'file-shared-before-you-joined';
|
||||
|
||||
-- Worker rows carried only `source: 'worker'` and a title, so the app they
|
||||
-- belong to is unrecoverable and they stay unattributed. The title prefix is
|
||||
-- the sole surviving signal of which way the deploy went.
|
||||
UPDATE `notification`
|
||||
SET `audience` = 'developer',
|
||||
`app_uid` = NULL,
|
||||
`type` = CASE
|
||||
WHEN JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.title')) LIKE 'Successfully deployed %'
|
||||
THEN 'app.worker.deployed'
|
||||
ELSE 'app.worker.deployFailed'
|
||||
END
|
||||
WHERE `type` = ''
|
||||
AND JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.source')) = 'worker';
|
||||
@@ -1,42 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Index `notification`.`created_at`. The retention sweep selects by age alone
|
||||
-- and by nothing else, so no existing index on the table narrows it — every
|
||||
-- pass would otherwise scan the whole table to find the oldest few hundred
|
||||
-- rows.
|
||||
--
|
||||
-- Idempotent: the guarded procedure, as mig_26. There is no per-file
|
||||
-- applied-state tracking, so a replay has to be a no-op.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_created_at_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_notification_created_at_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'notification'
|
||||
AND INDEX_NAME = 'idx_notification_created_at'
|
||||
) THEN
|
||||
ALTER TABLE `notification` ADD INDEX `idx_notification_created_at`
|
||||
(`created_at`);
|
||||
END IF;
|
||||
END //
|
||||
DELIMITER ;
|
||||
CALL _puter_add_notification_created_at_index();
|
||||
DROP PROCEDURE IF EXISTS _puter_add_notification_created_at_index;
|
||||
@@ -1,25 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Home region column. Mirrors SQLite migration 0074. Home region this
|
||||
-- account's data belongs in — a node id, not a storage region. NULL for
|
||||
-- accounts that predate the column, which fall back to `signup_server`.
|
||||
--
|
||||
-- Idempotent: the column add uses _puter_add_col (defined in mig_1, which
|
||||
-- leaves it resident for later migrations).
|
||||
|
||||
CALL _puter_add_col('user', 'home', '`home` VARCHAR(64) DEFAULT NULL');
|
||||
@@ -1,59 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Subscriptions that outlive the connection that made them. See
|
||||
-- sqlite/0075_event-subscriptions.sql for the column rationale.
|
||||
--
|
||||
-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline,
|
||||
-- as mig_23. There is no per-file applied-state tracking, so a replay has to
|
||||
-- be a no-op.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `event_subscriptions` (
|
||||
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
|
||||
`sub_id` varchar(80) NOT NULL,
|
||||
`token` varchar(255) NOT NULL,
|
||||
`owner_user_id` int unsigned NOT NULL,
|
||||
`holder_user_id` int unsigned NOT NULL,
|
||||
-- Matches `apps`.`uid` exactly, charset included, so an equality against
|
||||
-- one never falls back to a conversion. No foreign key: a row outlives the
|
||||
-- app that made it, which is what keeps it revocable afterwards.
|
||||
`app_uid` char(40) CHARACTER SET latin1 COLLATE latin1_swedish_ci
|
||||
DEFAULT NULL,
|
||||
`subject` varchar(4096) NOT NULL,
|
||||
`anchor_uid` char(36) NOT NULL,
|
||||
`anchor_path` varchar(4096) NOT NULL,
|
||||
`match` varchar(1024) DEFAULT NULL,
|
||||
`delivery` varchar(16) NOT NULL,
|
||||
`ops` varchar(64) DEFAULT NULL,
|
||||
`handler_name` varchar(128) DEFAULT NULL,
|
||||
`targets` json NOT NULL,
|
||||
`context` text,
|
||||
`permission` varchar(32) NOT NULL,
|
||||
`suspended_at` bigint DEFAULT NULL,
|
||||
`suspended_reason` varchar(64) DEFAULT NULL,
|
||||
`expires_at` bigint DEFAULT NULL,
|
||||
`created_at` bigint NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `idx_event_subscriptions_sub_id` (`sub_id`),
|
||||
KEY `idx_event_subscriptions_token` (`token`),
|
||||
KEY `idx_event_subscriptions_holder` (`holder_user_id`, `app_uid`),
|
||||
KEY `idx_event_subscriptions_owner` (`owner_user_id`),
|
||||
CONSTRAINT `fk_event_subscriptions_owner` FOREIGN KEY (`owner_user_id`)
|
||||
REFERENCES `user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
CONSTRAINT `fk_event_subscriptions_holder` FOREIGN KEY (`holder_user_id`)
|
||||
REFERENCES `user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
Loaded 100 of 2736 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user