mirror of
https://github.com/HeyPuter/puter.git
synced 2026-11-13 20:12:57 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Filip Kujundžić
|
62bc8d6d6f | ||
|
404oops
|
4700c2dc65 |
No files matched your search
@@ -16,8 +16,3 @@ MARIADB_PASSWORD=replace-with-strong-password
|
||||
S3_ACCESS_KEY=puter
|
||||
S3_SECRET_KEY=replace-with-strong-secret
|
||||
S3_BUCKET=puter-local
|
||||
|
||||
# ── Inbound SMTP (compose profile `smtp`) ------------------------------
|
||||
# Host port mapped to the receiver's 2525. 25 is what an MX record means;
|
||||
# many hosting providers block it.
|
||||
SMTP_PORT=25
|
||||
+46
-57
@@ -1,72 +1,61 @@
|
||||
# Default owner, including backend and SDK areas without a specialist below.
|
||||
|
||||
# The last matching rule takes precedence.
|
||||
|
||||
- @Salazareo
|
||||
* @Salazareo
|
||||
|
||||
# GUI
|
||||
|
||||
/src/gui/ @jelveh @Salazareo
|
||||
|
||||
# Documentation, developer site, and CLI
|
||||
|
||||
/doc/ @reynaldichernando @Salazareo
|
||||
/src/docs/ @reynaldichernando @Salazareo
|
||||
/src/dev-center/ @reynaldichernando @Salazareo
|
||||
/src/cli/ @reynaldichernando @Salazareo
|
||||
|
||||
# Backend AI
|
||||
|
||||
/src/backend/controllers/puterai/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/ai-\*/ @404oops @ProgrammerIn-wonderland @Salazareo
|
||||
/doc/ @reynaldichernando @Salazareo
|
||||
/src/docs/ @reynaldichernando @Salazareo
|
||||
/src/dev-center/ @reynaldichernando @Salazareo
|
||||
/src/cli/ @reynaldichernando @Salazareo
|
||||
|
||||
# Backend: AI, hosting, workers, networking, and WebDAV
|
||||
|
||||
/src/backend/controllers/hosting/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/webdav/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/wisp/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/util/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/workers/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/services/localworker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/stores/subdomain/ @ProgrammerIn-wonderland @Salazareo
|
||||
/extensions/workerSandbox\* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker-types/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/peer/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/puterai/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/hosting/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/webdav/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/wisp/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/ai-*/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/util/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/drivers/workers/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/services/localworker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/stores/subdomain/ @ProgrammerIn-wonderland @Salazareo
|
||||
/extensions/workerSandbox* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/worker-types/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/backend/controllers/peer/ @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# Backend: sharing, permissions, and teams
|
||||
|
||||
/src/backend/controllers/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/controllers/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/acl/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/apps/AppPermissionService\* @jfcastro92 @Salazareo
|
||||
/src/backend/stores/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/group/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/controllers/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/controllers/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/acl/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/team/ @jfcastro92 @Salazareo
|
||||
/src/backend/services/apps/AppPermissionService* @jfcastro92 @Salazareo
|
||||
/src/backend/stores/share/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/permission/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/group/ @jfcastro92 @Salazareo
|
||||
/src/backend/stores/team/ @jfcastro92 @Salazareo
|
||||
|
||||
# SDK: AI, hosting, workers, and networking
|
||||
|
||||
/src/puter-js/src/modules/ai/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/hosting/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Workers* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/networking/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/lib/networkUtils* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/ai.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/hosting.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/workers.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/net.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Peer\* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/ai/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/hosting/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Workers* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/networking/ @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/lib/networkUtils* @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/ai.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/hosting.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/workers.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/tests/api/suites/net.suite.ts @ProgrammerIn-wonderland @Salazareo
|
||||
/src/puter-js/src/modules/Peer* @ProgrammerIn-wonderland @Salazareo
|
||||
|
||||
# SDK: sharing and permissions
|
||||
|
||||
/src/puter-js/src/modules/perms/ @jfcastro92
|
||||
/src/puter-js/src/modules/FileSystem/operations/share* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/unshare* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/getShares* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/listShared* @jfcastro92 @Salazareo
|
||||
/src/puter-js/tests/api/suites/perms.suite.ts @jfcastro92 @Salazareo
|
||||
/src/puter-js/tests/api/suites/sharing.suite.ts @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/perms/ @jfcastro92
|
||||
/src/puter-js/src/modules/FileSystem/operations/share* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/unshare* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/getShares* @jfcastro92 @Salazareo
|
||||
/src/puter-js/src/modules/FileSystem/operations/listShared* @jfcastro92 @Salazareo
|
||||
/src/puter-js/tests/api/suites/perms.suite.ts @jfcastro92 @Salazareo
|
||||
/src/puter-js/tests/api/suites/sharing.suite.ts @jfcastro92 @Salazareo
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
|
||||
@@ -1,21 +1,17 @@
|
||||
name: Backend Tests
|
||||
|
||||
# Always triggers on PRs so it can be a required check; the `changes` job below
|
||||
# Always triggers so it can be a required PR check; the `changes` job below
|
||||
# decides whether backend/extension paths actually changed (test-only changes
|
||||
# match the same globs, since tests are co-located with the code), and `test`
|
||||
# still reports success when they didn't. Pushes to main produce the coverage
|
||||
# PRs compare against. puter.js SDK changes are covered by puterjs-tests.yaml.
|
||||
# match the same globs, since tests are co-located with the code), and every
|
||||
# other job is skipped — not absent — when they didn't, which still satisfies
|
||||
# a required check. puter.js SDK changes are covered by puterjs-tests.yaml.
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
# Reads main's coverage artifact for the PR comparison.
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
@@ -46,9 +42,12 @@ jobs:
|
||||
# The consuming repo runs an equivalent gate, but only once someone bumps the
|
||||
# submodule pointer — too late to keep the error off this repo's default
|
||||
# branch. Hence a gate here, on this repo's own pull requests.
|
||||
#
|
||||
# Its own job rather than a step in `test`: that one runs a base/PR matrix for
|
||||
# coverage comparison, and the check only needs the PR ref, once.
|
||||
typecheck:
|
||||
needs: changes
|
||||
if: github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true'
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -57,7 +56,7 @@ jobs:
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
@@ -66,146 +65,85 @@ jobs:
|
||||
- name: Type check (new errors only)
|
||||
run: npm run typecheck
|
||||
|
||||
# The suite is CPU-bound, so it's split across runners. Each shard writes a
|
||||
# blob report (results plus coverage) that the `coverage` job merges.
|
||||
test-shard:
|
||||
# `needs.changes` gates the steps below, not this job, on purpose: a
|
||||
# job-level `if` on a matrix job skips the whole job as one unexpanded
|
||||
# check ("test (${{ matrix.artifact }})", literally, since there's no
|
||||
# matrix context to fill it in) instead of the two real per-artifact
|
||||
# checks a required rule pins to ("test (base)" / "test (pr)") — so the
|
||||
# required check never gets reported and the PR blocks forever. Letting
|
||||
# the matrix always expand keeps the check names stable either way.
|
||||
test:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
name: test shard ${{ matrix.shard }}/${{ strategy.job-total }}
|
||||
# Overrides the default matrix naming, which would otherwise bake the
|
||||
# base/head branch name (matrix.ref) into the check name — making it a
|
||||
# different string on every PR and impossible to pin as a required check.
|
||||
name: test (${{ matrix.artifact }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
include:
|
||||
- ref: ${{ github.base_ref }}
|
||||
artifact: base
|
||||
- ref: ${{ github.head_ref }}
|
||||
artifact: pr
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
- name: Checkout ${{ matrix.ref }}
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
ref: ${{ matrix.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
|
||||
- name: Set up Node.js
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Run backend tests with coverage
|
||||
run: >-
|
||||
npm run test:backend --
|
||||
--coverage
|
||||
--shard=${{ matrix.shard }}/${{ strategy.job-total }}
|
||||
--reporter=default
|
||||
--reporter=blob
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload blob report
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: backend-blob-${{ matrix.shard }}
|
||||
path: .vitest-reports/
|
||||
include-hidden-files: true
|
||||
retention-days: 1
|
||||
|
||||
# Required check, so its PR name is pinned. A skipped job would satisfy it,
|
||||
# so this always runs and fails on any shard that didn't pass.
|
||||
test:
|
||||
needs: [changes, test-shard]
|
||||
if: ${{ !cancelled() }}
|
||||
name: test (${{ github.event_name == 'push' && 'main' || 'pr' }})
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check shard results
|
||||
env:
|
||||
CHANGES_RESULT: ${{ needs.changes.result }}
|
||||
BACKEND_CHANGED: ${{ needs.changes.outputs.backend }}
|
||||
SHARDS_RESULT: ${{ needs.test-shard.result }}
|
||||
run: |
|
||||
if [ "$CHANGES_RESULT" != "success" ]; then
|
||||
echo "changes job: $CHANGES_RESULT"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$BACKEND_CHANGED" != "true" ]; then
|
||||
echo "No backend changes."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$SHARDS_RESULT" != "success" ]; then
|
||||
echo "test shards: $SHARDS_RESULT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
coverage:
|
||||
needs: [changes, test-shard]
|
||||
if: ${{ !cancelled() && needs.test-shard.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: backend-blob-*
|
||||
path: .vitest-reports
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge shard coverage
|
||||
run: npx vitest run --config src/backend/vitest.config.ts --merge-reports --coverage
|
||||
if: needs.changes.outputs.backend == 'true'
|
||||
run: npm run test:backend -- --coverage
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: needs.changes.outputs.backend == 'true' && !cancelled()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: backend-coverage-${{ github.event_name == 'push' && 'main' || 'pr' }}
|
||||
name: backend-coverage-${{ matrix.artifact }}
|
||||
path: src/backend/coverage/
|
||||
retention-days: ${{ github.event_name == 'push' && 30 || 14 }}
|
||||
retention-days: 14
|
||||
|
||||
# Latest main coverage from this repo's own runs; a fork can't publish
|
||||
# under this repository id.
|
||||
- name: Download main coverage
|
||||
id: base
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
id=$(gh api "repos/$REPO/actions/artifacts?name=backend-coverage-main&per_page=20" --jq '
|
||||
[.artifacts[]
|
||||
| select(.expired | not)
|
||||
| select(.workflow_run.head_branch == "main")
|
||||
| select(.workflow_run.head_repository_id == .workflow_run.repository_id)
|
||||
][0].id // empty')
|
||||
if [ -z "$id" ]; then
|
||||
echo "No main coverage artifact; reporting without a comparison."
|
||||
exit 0
|
||||
fi
|
||||
gh api "repos/$REPO/actions/artifacts/$id/zip" > coverage-base.zip
|
||||
unzip -q coverage-base.zip -d coverage-base
|
||||
echo "found=true" >> "$GITHUB_OUTPUT"
|
||||
report-coverage:
|
||||
needs: [changes, test]
|
||||
if: always() && needs.changes.outputs.backend == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Download PR coverage
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: backend-coverage-pr
|
||||
path: src/backend/coverage
|
||||
|
||||
- name: Download base coverage
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: backend-coverage-base
|
||||
path: coverage-base
|
||||
|
||||
- name: Report coverage on PR
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: davelosert/vitest-coverage-report-action@v2
|
||||
with:
|
||||
json-summary-path: src/backend/coverage/coverage-summary.json
|
||||
json-final-path: src/backend/coverage/coverage-final.json
|
||||
json-summary-compare-path: ${{ steps.base.outputs.found == 'true' && 'coverage-base/coverage-summary.json' || '' }}
|
||||
json-summary-compare-path: coverage-base/coverage-summary.json
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
if: needs.changes.outputs.puterjs == 'true'
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '26'
|
||||
node-version: '24'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
|
||||
@@ -66,7 +66,6 @@ AGENTS.md
|
||||
|
||||
|
||||
coverage/
|
||||
.vitest-reports/
|
||||
*.log
|
||||
undefined
|
||||
servers.json
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for AI coding agents working in this repository, and the source of truth for how we write code, tests, and docs here. Read it before making changes. FOLLOW GUIDANCE AS CLOSELY AS POSSIBLE. If you think a rule is wrong, raise an issue or flag a maintainer instead of doing what you think is right.
|
||||
Guidance for AI coding agents working in this repository. Read this before making changes. FOLLOW GUIDANCE AS CLOSELY AS POSSIBLE. If you think the guidance is wrong, raise an issue or flag a maintainer — don't just do what you think is right. This is the source of truth for how we write code, tests, and docs in this repo.
|
||||
|
||||
## Documentation Index
|
||||
|
||||
Use these as the source of truth before exploring further:
|
||||
|
||||
- [README.md](README.md) — project overview and quickstart.
|
||||
- [doc/architecture.md](doc/architecture.md) — backend layered stack (controllers → drivers → services → stores → clients), `PuterServer` wiring, `Context` (ALS), actors, and extensions.
|
||||
- [doc/architecture.md](doc/architecture.md) — backend layered stack (controllers → drivers → services → stores → clients), `PuterServer` wiring, `Context` (ALS), and extensions.
|
||||
- [doc/contributing-apis.md](doc/contributing-apis.md) — adding and maintaining public APIs end to end (backend surface → puter.js → types → docs → tests). Follow it for any API work.
|
||||
- [doc/pagination.md](doc/pagination.md) — the one pagination convention for list APIs (limit/cursor/offset/includeTotal, envelope shape, cursor semantics).
|
||||
- [doc/alarms.md](doc/alarms.md) — raising alarms and picking a severity (what pages, what only gets recorded), plus the config that routes them.
|
||||
- [doc/self-hosting.md](doc/self-hosting.md) — running Puter outside hosted infra.
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) — contributor entry point and PR conventions.
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) — testing, security, AI-assisted code, PR conventions, Boy Scout Rule.
|
||||
- [SECURITY.md](SECURITY.md) — how to report vulnerabilities (do not file them publicly).
|
||||
- [BUG-BOUNTY.md](BUG-BOUNTY.md) — bounty program scope.
|
||||
- [TRADEMARK.md](TRADEMARK.md) — trademark usage.
|
||||
@@ -26,16 +26,16 @@ These apply everywhere — backend, puter.js, and GUI.
|
||||
### Language & files
|
||||
|
||||
- Write ES modules, not CommonJS — we transpile and build as needed.
|
||||
- TypeScript preferred for new files in the backend and extensions; convert existing JS there opportunistically when you're already touching a file. The GUI and puter.js are plain JavaScript — don't introduce TypeScript files in them.
|
||||
- In plain-JS files, typing with JSDoc (`@type`/`@param`/`@returns`, `@typedef` for shared shapes, using the TypeScript type system) is encouraged. API types must not be `unknown` or untyped `...args`: spell out the real parameter and return shapes, unless the value passes through transparently to an upstream layer that owns its type. puter.js has stricter rules ([below](#types)).
|
||||
- Make new types findable: descriptive `PascalCase` names, exported from the obvious entry point. A type used from many places goes in the owning module's `types.ts` (e.g. [src/backend/controllers/types.ts](src/backend/controllers/types.ts)); a type with a single consumer can stay next to it.
|
||||
- Naming: `camelCase` for variables and functions, `PascalCase` for classes and files containing a class (`AuthService.ts`). **Prefer `camelCase` for all new code**, including parameters and internal object properties. `snake_case` is reserved for external contracts that already use it: wire/JSON keys, stable API error codes, public option names that already ship as `snake_case`, and established `puter_*` namespaced fields. Don't rename those (it's a breaking change), and don't add new `snake_case` beside them; a camelCase local can carry a value into a wire key (`{ operation_id: operationId }`).
|
||||
- TypeScript preferred for new files in the backend and extensions; convert existing JS there opportunistically when you're already touching a file. The GUI and puter.js are plain JavaScript — don't introduce TypeScript files in them. In puter.js, type with JSDoc instead (see the puter.js section for what must be typed).
|
||||
- **Reuse before adding.** Search for an existing type, helper, or implementation and extend it; only add a new one when nothing suitable exists.
|
||||
- Make new types findable: descriptive `PascalCase` names, exported from the obvious entry point. A type used from many places belongs in the owning module's `types.ts` (e.g. [src/backend/controllers/types.ts](src/backend/controllers/types.ts)) rather than bloating a logic file; a type with a single consumer can stay next to it.
|
||||
- Naming: `camelCase` for variables/functions, `PascalCase` for classes and files containing a class (`AuthService.ts`). **Prefer `camelCase` for all new code** — new local variables, parameters, functions, and internal object properties. `snake_case` is reserved for identifiers that are part of an external contract: wire/JSON keys sent to or received from the backend, stable API error codes, public option names that already ship as `snake_case`, and established `puter_*` namespaced fields. Don't rename those (it's a breaking change), but don't introduce new `snake_case` alongside them either — a camelCase local can carry a value into a `snake_case` wire key (`{ operation_id: operationId }`).
|
||||
|
||||
### Comments
|
||||
|
||||
Prefer self-documenting code. Comment only when the _why_ is non-obvious or a usage detail would trip the next reader. **A comment is a sentence or two**: one or two lines of `//`, a few of `/** ... */` JSDoc. State the constraint and stop. Don't restate the code, narrate history ("used to swallow the failure, which meant…"), argue the design, or explain parameters the signature already shows. Reasoning that needs paragraphs belongs in a design doc.
|
||||
Keep comments light; prefer self-documenting code. Comment only when the _why_ is non-obvious or a usage detail would trip the next reader. Use `//` for single lines and `/** ... */` JSDoc when it genuinely needs more — if a comment runs long, it's probably too long. Don't restate the code and don't reference the current task, PR, or version — those rot. **No ticket references** (`PUT-1234`, `// fix for FOO-99`) in code, comments, or test names; describe the why in domain terms, not project-management terms. Use plain ASCII `-` in comment section dividers (`// -- Section --`), never box-drawing characters.
|
||||
|
||||
Don't reference the current task, PR, or version; those rot. **No ticket references** (`PUT-1234`, `// fix for FOO-99`) in code, comments, or test names: describe the why in domain terms. Section dividers use plain ASCII (`// -- Section --`), never box-drawing characters.
|
||||
**Be concise — a comment is a sentence or two, not a paragraph block.** One or two lines for `//`, a few for JSDoc. Don't narrate the history ("used to swallow the failure, which meant…"), don't argue the design at length, and don't explain each parameter when the signature already does. State the constraint and stop. If the reasoning genuinely needs paragraphs, it belongs in a design doc, not above the function.
|
||||
|
||||
### Security & privacy
|
||||
|
||||
@@ -47,59 +47,77 @@ Before opening a PR, scan the diff for:
|
||||
|
||||
When in doubt, return less. Auth-, permission-, or data-export-related changes deserve an explicit callout in the PR description.
|
||||
|
||||
Commit messages and PR descriptions never credit anyone or mention a bug report. For security fixes, state what the change does mechanically ("scoped tokens do xyz now"), not the vulnerability or how it could be abused.
|
||||
|
||||
### Reuse and abstraction
|
||||
|
||||
- **Search before you build.** Before writing a type, helper, cache/TTL map, retry, lock, pagination, validator, serializer, error mapper, host/email normalizer, or provider adapter, grep `src/backend/util/`, the owning layer, and `extensions/` for one that exists, and extend it instead of adding a parallel copy. If you find two copies, consolidate them or flag it. Before opening the PR, check the diff for new helpers that duplicate existing ones.
|
||||
- **One path per operation.** Single-item variants call the batch version. Legacy and v2 endpoints share one service method. A second entry point (OIDC, WebDAV, admin, cron) calls the same service as the first instead of re-implementing it.
|
||||
- **No abstraction without a second caller.** No base class, interface, factory, registry, decorator, or option bag with one implementation. No wrappers that only forward, no interface methods nobody calls, no methods only tests call.
|
||||
- **Variants are data.** Providers or models that differ only in constants (base URL, prices, key names) are entries in a table read by one implementation, not one file each.
|
||||
|
||||
### Hot paths
|
||||
|
||||
- No per-row queries or awaits in loops: use the store's batch method (add one if missing) or [src/backend/util/concurrency.ts](src/backend/util/concurrency.ts).
|
||||
- Bound everything: queries get `LIMIT`/pagination, in-memory maps get eviction, outbound calls get a timeout and a size cap.
|
||||
- Build clients, catalogs, and config-derived lookups once, not per request.
|
||||
- Listeners on fleet-wide events (`fs.write.file`, route lifecycle) return before any I/O for requests they don't concern.
|
||||
|
||||
### Working rules of thumb
|
||||
|
||||
- **Run it, don't just compile it.** "It type-checks" is not "it works." Exercise the code path end-to-end at least once.
|
||||
- **Read the neighbors before writing.** Match the shape of similar things already in the tree. If you think the existing pattern is wrong, raise it — don't quietly diverge.
|
||||
- **Test new behavior.** Every new function, endpoint, driver method, or logic branch gets a test; every bug fix gets a regression test that fails before the fix. If something is genuinely hard to test, skip it but say so in the PR.
|
||||
- **Boy Scout Rule, proportional to the change.** In files you're already touching, fix the typo, the dead import, the missing test, the bit you had to read twice. Don't ride a refactor along with a bug fix.
|
||||
- **Understand what you commit.** AI assistance is fine; shipping code you couldn't have written, debugged, or defended in review is not.
|
||||
- **Boy Scout Rule, proportional to the change.** Fix the obvious typo or dead import in files you're already touching; don't ride a refactor along with a bug fix.
|
||||
- **Understand what you commit.** AI assistance is fine; shipping code you couldn't defend in review is not.
|
||||
- **A limit change is not done until the docs change with it.** Every rate limit, concurrency cap, quota, and allowance in the code is published in [src/docs/src/rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) — an undisclosed limit is one developers discover as a service failure. If a PR moves any of these numbers, the same PR updates that page. The numbers live in `src/backend/controllers/fs/limits.ts` (filesystem), `src/backend/drivers/util/aiLimits.ts` (all AI drivers), per-driver `rateLimit`/`concurrent` configs, and `src/backend/data/subPolicies/` (free-tier allowances).
|
||||
|
||||
---
|
||||
|
||||
## Local development
|
||||
## Local development configuration
|
||||
|
||||
To test a local Docker build, use the gitignored `docker-compose.override.yml` described in [Self-Hosting → Building from source](doc/self-hosting.md#building-from-source-instead-of-pulling). Keep local build settings there, not in `docker-compose.yml`.
|
||||
To test a local Docker build, create `docker-compose.override.yml` in the repository
|
||||
root (or merge into the existing file):
|
||||
|
||||
When budget checks block unrelated local API tests, you may set `"unlimitedMetering": true` without further approval. Merge it into the gitignored `config.json` (`npm start`) or `puter/config/config.json` (Docker) and restart Puter. It covers every account on that instance, guests included, and usage is still recorded. Keep normal settings when testing budget or subscription enforcement, and leave shared defaults unchanged.
|
||||
```yaml
|
||||
services:
|
||||
puter:
|
||||
pull_policy: never
|
||||
build:
|
||||
context: .
|
||||
```
|
||||
|
||||
Compose merges it automatically. Run `docker compose up -d --build` to build and
|
||||
start the stack, and repeat after source changes. Keep local build settings in
|
||||
this gitignored override instead of editing `docker-compose.yml`, so they stay
|
||||
out of pull requests and do not conflict when pulling updates. See
|
||||
[Self-Hosting Puter](doc/self-hosting.md#building-from-source-instead-of-pulling)
|
||||
for the required runtime configuration.
|
||||
|
||||
For local API testing, agents may decide whether to set `"unlimitedMetering": true`
|
||||
when budget checks block unrelated tests; no additional approval is needed.
|
||||
Merge it into the ignored `config.json` for `npm start` or
|
||||
`puter/config/config.json` for Docker, then restart Puter. It applies to all
|
||||
accounts on that local instance, including guests, and usage is still recorded.
|
||||
Keep normal settings when testing budget or subscription enforcement, and leave
|
||||
shared defaults unchanged.
|
||||
|
||||
---
|
||||
|
||||
## Backend
|
||||
|
||||
Controllers → drivers (optional) → services → stores → clients, all configured from `IConfig`. Each layer depends only on the layers beneath it and receives them through its constructor; `PuterServer` ([src/backend/server.ts](src/backend/server.ts)) wires them. [doc/architecture.md](doc/architecture.md) is the full reference.
|
||||
A layered stack with explicit dependency injection: each layer depends only on the layers beneath it, receives them through its constructor, and `PuterServer` ([src/backend/server.ts](src/backend/server.ts)) wires the whole thing together. [doc/architecture.md](doc/architecture.md) is the full reference.
|
||||
|
||||
A public API can be a controller or a driver. Prefer a controller when you need fine-grained control over routes and gates; [doc/contributing-apis.md](doc/contributing-apis.md#controller-or-driver) has the decision guide.
|
||||
| Layer (top → bottom) | Lives in | Responsibility |
|
||||
| -------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| **Controllers** | [src/backend/controllers/](src/backend/controllers/) | Route handlers: parse/validate input, per-route gates via `RouteOptions` (auth, subdomain, rate limit, body parsers), call services, format responses. |
|
||||
| **Drivers** | [src/backend/drivers/](src/backend/drivers/) | Optional RPC-style handlers on `/drivers/*`; thin shells that validate inputs and call services/stores. |
|
||||
| **Services** | [src/backend/services/](src/backend/services/) | Business logic. Assume the caller is already authenticated/authorized. |
|
||||
| **Stores** | [src/backend/stores/](src/backend/stores/) | Persistence; wraps clients in the domain shapes services consume. |
|
||||
| **Clients** | [src/backend/clients/](src/backend/clients/) | Adapters for external/internal services (sql, redis, s3, email, event bus). Protocols, not domain concepts. |
|
||||
| **Config** | `config.*.json` → `IConfig` | Flat, typed config object every layer receives at construction. |
|
||||
|
||||
Layer rules:
|
||||
A public API can be exposed through either a controller or a driver — both are supported; prefer a controller when you need fine-grained control over routes and gates. [doc/contributing-apis.md](doc/contributing-apis.md) has the decision guide with links to the decorators and middleware for each.
|
||||
|
||||
- **Don't reach across layers.** Controllers don't poke clients; services don't register routes; lower layers never import from controllers. If you want to, the abstraction is wrong; fix the abstraction.
|
||||
- **Gates run at the edge.** Controllers and drivers validate input and apply auth gates; services assume the caller is already authenticated and authorized.
|
||||
Cross-layer rules:
|
||||
|
||||
- **Don't reach across layers.** Controllers don't poke clients; services don't register routes. If you want to, the abstraction is wrong — fix the abstraction.
|
||||
- **Don't call sideways within a layer for code reuse.** Two services needing the same logic means a util/helper, not a service-to-service dependency.
|
||||
- **A table's store owns every query against it.** Services, controllers, and extensions don't run raw SQL/redis against a table that has a store; add the store method so cache invalidation stays with the write.
|
||||
- **Drivers and controllers stay thin.** Domain rules a second entry point needs (signup, app CRUD, worker deploy, AI routing/billing) live in a service, not in the first driver or controller that needed them.
|
||||
- **Prefer explicit arguments over [`Context`](src/backend/core/context.ts) (ALS).** Use it only for request-scoped values that would otherwise thread through many layers; today that's mostly `actor` and `req`.
|
||||
- **Prefer explicit arguments over `Context` (ALS).** Reach for [Context](src/backend/core/context.ts) only for genuinely request-scoped values that would otherwise thread through many layers — today mostly `actor` and `req`.
|
||||
|
||||
### Extensions
|
||||
|
||||
[extensions/](extensions/) is for **non-crucial parts of the system**: Puter must still work with the extension removed. If core needs to call it, it belongs in core ([whoami](extensions/whoami.ts) is the cautionary example). Inside an extension, follow the same layered structure, unless it only needs a few route handlers, in which case the `extension.get/post/...` helpers are enough on their own. The `extension` API is in [doc/architecture.md](doc/architecture.md#extension-api).
|
||||
[extensions/](extensions/) parallels the layered stack and is for **non-crucial parts of the system** — Puter still works with the extension removed. If core needs to call it, it belongs in core, not an extension (see [whoami](extensions/whoami.ts) as the cautionary example). The `extension` global ([src/backend/extensions.ts](src/backend/extensions.ts)) exposes:
|
||||
|
||||
- `extension.registerClient/Store/Service/Driver/Controller(name, Class)` for first-class additions.
|
||||
- `extension.on(event, handler)` and `extension.get/post/put/delete/patch/use(path, opts?, handler)` for the lightweight common case. `opts` is the same `RouteOptions` controllers use.
|
||||
- `extension.import('client' | 'store' | 'service' | 'controller' | 'driver')` — lazy proxy to instantiated core objects. `extension.config` exposes live config.
|
||||
|
||||
Follow the same layered structure inside an extension — unless it only needs a few route handlers, in which case the lightweight helpers are enough on their own.
|
||||
|
||||
### Backend tests
|
||||
|
||||
@@ -112,84 +130,44 @@ Layer rules:
|
||||
|
||||
## puter.js (the SDK)
|
||||
|
||||
[src/puter-js/](src/puter-js/) is the public SDK. It ships live from `https://js.puter.com/v2/` with no version pinning, so every change reaches every existing app at once; see [Rule zero](doc/contributing-apis.md#rule-zero-dont-break-callers).
|
||||
[src/puter-js/](src/puter-js/) is the public SDK. It ships live from `https://js.puter.com/v2/` with no version pinning — every existing app picks up changes immediately. Treat every observable behavior (signatures, response fields, error codes) as something a production app depends on.
|
||||
|
||||
Layout: SDK modules in [src/puter-js/src/modules/](src/puter-js/src/modules/) (one file or directory per area — `FileSystem/`, `kv/`, `ai/`, …), shared plumbing in [src/puter-js/src/lib/](src/puter-js/src/lib/), generated (gitignored) type declarations in `src/puter-js/types/`, API tests in [src/puter-js/tests/api/](src/puter-js/tests/api/), UI e2e tests in [src/puter-js/tests/e2e/](src/puter-js/tests/e2e/), developer docs in [src/docs/](src/docs/).
|
||||
|
||||
Every SDK change carries all five of these, or the PR is incomplete. [doc/contributing-apis.md](doc/contributing-apis.md) has the steps for each.
|
||||
Language & typing: puter.js source is plain JavaScript — never TypeScript files — typed via JSDoc. **The JSDoc is the source of truth for the SDK's types.** `src/puter-js/types/` is `tsc --emitDeclarationOnly` output: the SDK build (`npm run build` in `src/puter-js`) generates it, the npm tarball ships it so TypeScript consumers get declarations, and git ignores it — it is never committed and never edited by hand. `npm run check:puterjs:types` generates it in CI and type-checks the result without `skipLibCheck`. The one hand-written declaration file is [src/puter-js/index.d.ts](src/puter-js/index.d.ts), which decides what is public and re-exports the generated names.
|
||||
|
||||
1. **Backward compatibility**, unless a maintainer explicitly signs off on a break ([rules](doc/contributing-apis.md#maintaining-an-existing-api)).
|
||||
2. **Tests** in the cross-platform API suite, or the Playwright harness for desktop-rendered `puter.ui.*` ([steps](doc/contributing-apis.md#6-tests)).
|
||||
3. **Docs** in [src/docs/src/](src/docs/src/) that match the implementation exactly ([steps](doc/contributing-apis.md#5-docs), [writing rules](#developer-docs)).
|
||||
4. **Types**, per the rules below.
|
||||
5. **Error handling**: `{ message, code }` with stable `snake_case` codes, as in [modules/kv/](src/puter-js/src/modules/kv/) ([steps](doc/contributing-apis.md#3-puterjs)).
|
||||
|
||||
### Types
|
||||
|
||||
puter.js source is plain JavaScript typed via JSDoc, and **the JSDoc is the source of truth for the SDK's types**. `src/puter-js/types/` is `tsc --emitDeclarationOnly` output: the SDK build (`npm run build` in `src/puter-js`) generates it, the npm tarball ships it, and git ignores it. Never commit or hand-edit it. `npm run check:puterjs:types` generates it and type-checks the result without `skipLibCheck`; run it after any type change. The one hand-written declaration file is [src/puter-js/index.d.ts](src/puter-js/index.d.ts): it decides what is public and re-exports the generated names, so name a new type there if consumers should be able to import it.
|
||||
|
||||
- Public (exposed) methods carry JSDoc types for parameters and return value, with one `@overload` block per accepted call form; those overloads _are_ the published signature. Typing private helpers is at your discretion; annotate where it helps the next reader.
|
||||
- Tag members `@internal`, not `@private`, to strip them from the generated declarations.
|
||||
- Declarations must match runtime behavior exactly. A wrong type is worse than a missing one.
|
||||
|
||||
Declare a shape where it belongs and reference it elsewhere with an `import(...)` type. A shape more than one file needs lives in the module's `types.js`; one shared across modules lives in [src/puter-js/src/lib/types.js](src/puter-js/src/lib/types.js); one with a single consumer can stay next to it:
|
||||
Declare a shape where it belongs, and reference it with an `import(...)` type from elsewhere. A shape more than one file needs lives in the module's `types.js`; a shape shared across modules lives in [src/puter-js/src/lib/types.js](src/puter-js/src/lib/types.js); a shape with one consumer can stay next to it:
|
||||
|
||||
```js
|
||||
/** @typedef {import('./types.js').KVOptConfig} KVOptConfig */
|
||||
```
|
||||
|
||||
Use `@typedef {Object}` + `@property` for any shape whose fields need documenting; it's the only JSDoc form that carries a per-field doc comment into the generated declarations. Use the inline form for small internal shapes with nothing to say per field, and prefer `unknown` over `*`:
|
||||
Use `@typedef {Object}` + `@property` for any shape whose fields need documenting — it is the only JSDoc form that carries a doc comment per field into the generated declaration. Keep the inline object-literal form for small internal shapes with nothing to say about each field, and prefer `unknown` over `*`:
|
||||
|
||||
```js
|
||||
/** @typedef {{ key: string; value: unknown }} KVEntry */
|
||||
```
|
||||
|
||||
---
|
||||
Public (exposed) methods must carry JSDoc types — parameters and return value — with one `@overload` block per accepted call form, since those overloads _are_ the published signature. Unexposed/private helpers are typed at the contributor's discretion: annotate where it helps the next reader, and either way keep them clean. Members tagged `@internal` are stripped from the generated declarations, so use that tag rather than `@private` to keep something off the public surface.
|
||||
|
||||
## Developer docs
|
||||
Typing in JS files is encouraged: annotate with JSDoc `@type`/`@param`/`@returns` using the TypeScript type system, and define shared shapes with `@typedef`. API types must not be `unknown` or untyped `...args` — spell out the real parameter and return shapes; the only exception is values passed through transparently to an upstream layer that owns their type. For example:
|
||||
|
||||
[src/docs/src/](src/docs/src/) is the published puter.js documentation: one reference page per method (`<Area>/<method>.md`, [steps](doc/contributing-apis.md#5-docs)) and recipes in [src/docs/src/recipes/](src/docs/src/recipes/).
|
||||
```js
|
||||
/** @typedef {{ key: string; value: unknown }} KVEntry */
|
||||
|
||||
### Page structure
|
||||
/** @type {KVEntry[]} */
|
||||
let entries = [];
|
||||
```
|
||||
|
||||
The reference has two kinds of page:
|
||||
Every SDK change carries all five of the following — a puter.js PR missing one is incomplete:
|
||||
|
||||
- **API overview pages** (`<Area>.md`, such as [KV.md](src/docs/src/KV.md)) give a quick top-level picture of what the API is for and what it can do, with short examples and a list of its functions that links every function page in the area. Leave out the details. The first sentence of the intro says what the API is about. This is a judgment call too: when the API depends on a Puter concept the reader may not know, such as Teams, the first sentence can name it and the sentences after it explain it.
|
||||
- **Function pages** (`<Area>/<method>.md`) hold the details: parameters, defaults, return values, errors, limits, and edge cases.
|
||||
1. **Backward compatibility.** Mandatory unless a maintainer explicitly signs off on a break. Existing call signatures keep working (including both positional and options-object forms where a method supports them); new parameters are optional with defaults that preserve old behavior; never rename or repurpose existing params, response fields, or error codes. New parameter names are `camelCase` (existing `snake_case` stays for compatibility). Say in the PR how existing callers are unaffected.
|
||||
2. **Tests.** Add or extend a suite in [tests/api/suites/](src/puter-js/tests/api/suites/) (`<area>.suite.ts`; register new suites in `suites/index.ts` — no globbing). One suite runs unchanged on node, browser, and workerd via `npm run test:puterjs`; never write a per-platform test. The runners execute the **built** bundle — run `npm run build:workerLib` after SDK changes or the suite silently tests stale code. For `puter.ui.*` methods rendered by the desktop, use the Playwright e2e harness instead — see [src/puter-js/TESTING.md](src/puter-js/TESTING.md).
|
||||
3. **Docs.** New or changed APIs update [src/docs/src/](src/docs/src/): the method page (`<Area>/<method>.md`, with frontmatter and a runnable example) and the area overview when the surface changes. Docs are the contract users code against — signatures, defaults, and return shapes must match the implementation exactly.
|
||||
4. **Types.** Type the change in JSDoc on the implementation, then run `npm run check:puterjs:types` to confirm it still produces declarations that type-check. Name any new type in [src/puter-js/index.d.ts](src/puter-js/index.d.ts) if consumers should be able to import it. Declarations must match runtime behavior exactly — a wrong type is worse than a missing one.
|
||||
5. **Error handling.** Reject/throw `{ message, code }` objects with stable `snake_case` codes, matching the existing modules (see `KV.js`). Validate cheap preconditions client-side before making the network call; pass backend errors through unchanged rather than swallowing or re-wrapping them. Error codes are API surface — changing one is a breaking change.
|
||||
|
||||
An overview page answers "what is this API and is it what I need?", and a function page answers "how exactly does this call behave?". A change to one function usually only touches that function's page, so a new `kv.list()` option goes in `KV/list.md` without a matching update to `KV.md`. Don't copy a function's options, limits, or edge cases onto the overview page for completeness. Where a detail goes is a judgment call, not a hard rule: update the overview when a change affects how someone would describe the API as a whole.
|
||||
|
||||
When you add a page, add it to [src/docs/src/sidebar.js](src/docs/src/sidebar.js) under its area, or it won't show up in the navigation. When you add a runnable example, add its HTML to `src/docs/src/playground/examples/<slug>.html`, register the slug in [src/docs/src/examples.js](src/docs/src/examples.js), and link the code block to it by putting the slug after the language with a semicolon (` ```html;kv-set `).
|
||||
|
||||
### Writing docs
|
||||
|
||||
These apply to reference pages and recipes alike.
|
||||
|
||||
- **Document the developer-facing side only.** The reader is building on top of puter.js, so cover what affects their code: signatures, defaults, return shapes, errors, limits, and behavior they can observe. Leave out internal workings (which service handles the call, storage layout, caching, retries) unless they change what the developer has to do. [rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) is the exception: it publishes every limit, including timeouts and caps that come from how the backend works.
|
||||
- **Describe the current state.** Docs aren't a changelog. Write how the API behaves now, without "now", "no longer", "existing code is unaffected", or other wording that compares it to an older version.
|
||||
- **Verify before you document.** Run the calls you describe against a running Puter instead of inferring behavior from the source.
|
||||
- **Write plainly.** Use the literal words a developer would say out loud, with contractions and everyday words. No idioms or figurative verbs ("walk away from something shared with you" → "remove your own access", "nothing to page through" → "without pagination"). No formal register ("present only while more pages remain" → "includes a `cursor` if there are more pages to load"). Prefer dev terms such as atomic, concurrent, and insertion order.
|
||||
- **State the rule, then its consequence, in short sentences.** No metaphor in place of a rule. Name a concrete action ("you can create any files and folders inside it") over an abstract permission ("yours to organize freely"). Cut a sentence that only repeats the next one.
|
||||
- **Describe behavior directly.** No quoted inner-voice phrases. Write "`isNew` is `true` when the person did not have access before", not "this tells 'I just gave this to someone' from 'they already had it'".
|
||||
- **Keep references easy to follow.** Name what a list or code block belongs to in the sentence directly above it. Name a thing once and keep that name. Introduce an actor as "a user" before giving them a role: "A user can create a team. The user who creates the team becomes its owner."
|
||||
- **Sentence mechanics.** Never start a sentence with inline code: write "To store data, use the [`set()`](/KV/set/) method." Link every inline mention of an API to its page, every time. No em dashes, and no colons or semicolons mid-sentence; a colon that introduces a code block is fine.
|
||||
- **Show the correct way.** No "don't" and no wrong-way-first examples, in recipes and playground examples alike. Mention a wrong approach only when developers already hit it in practice, and state how it fails as a fact: "`add()` reads a plain object as a path map, so the call rejects with …".
|
||||
|
||||
### Recipes
|
||||
|
||||
Recipes are tested, copy-pasteable patterns for a task, one layer above the per-method reference, often combining several APIs. Agents find them through the `## Recipes` section of `llms.txt`.
|
||||
|
||||
- **Files.** One `.md` per recipe in [src/docs/src/recipes/](src/docs/src/recipes/), with `title`, `description`, `tags`, and optional `order` front matter; they're picked up automatically. Tags must exist in [src/docs/src/recipe-tags.js](src/docs/src/recipe-tags.js) or the build fails. Quote a `description` that contains a colon followed by a space. Every top-level `.md` in that directory is built as a recipe, so keep other markdown out of it.
|
||||
- **Title.** Short, Title Case, starts with a verb, and leaves out "to Your App" ("Add Team Support", "Deploy Workers Programmatically"). The slug is the title in kebab case, with no API-name prefix. In `llms.txt` an agent sees only the title and description before deciding to open a recipe, so word them to match the task the reader is trying to do.
|
||||
- **Description.** Written for someone who has never heard of Puter. Start with "Learn how to", put the action first, and say what the reader's app gains: "Learn how to add file sharing to your app with Puter.js, so users can share files in their Puter account with other people." No API names, no mechanism, no list of example cases, and no "not just …" contrast.
|
||||
- **Intro.** Explain the situation and why you'd do this, tied to something the reader already knows ("`puter.kv` is your database in Puter.js"). No "Use this when" bullet list.
|
||||
- **Structure.** Get to the first working call as early as possible, then add validation, rate limits, and sign-in after it. Explain a Puter concept the first time it appears, such as what `me.puter` is or that `puter.workers.exec()` opens the Puter sign-in window. Split recipes by what the reader needs, not by technique, and give a family of recipes a shared tag.
|
||||
- **Reading order.** A recipe reads top to bottom. Each code block uses only functions and variables from blocks above it, so the reader never has to search further down the page to understand it. Start with a small version that works, and have each section add one case. When a section changes a function from earlier, show the whole new version of that function there. If the code is built up over several sections, end with a `## Full Code` block that has all of it.
|
||||
- **Link to the reference for full option lists.** Mention only the options the recipe uses, then link to the function page for the rest ("All the sort options are listed on [`puter.fs.readdir()`](/FS/readdir/)."). A copied list of options, rules, or scopes goes stale when the reference changes.
|
||||
- **Limits.** Mention a limit only when it's the reason for a pattern the recipe uses, such as one wildcard subscription instead of one per key, and link to [rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md) or the function page for the number. Leave out limits that code following the recipe never reaches.
|
||||
- **Code.** Platform-neutral: just the code, with top-level `await`. No `<script>` or `<html>` wrapper, no IIFE.
|
||||
- **Tradeoffs.** Justify a choice with one scenario the reader can picture right away, in about two short paragraphs, or fold each point into the section where it matters. The description states the benefit only.
|
||||
- **Notes.** Leave out `## Notes` by default. Use it only to point to an architecturally different approach. A limit the recipe needs goes in the section it applies to, and other facts belong on the linked reference page.
|
||||
[doc/contributing-apis.md](doc/contributing-apis.md) walks the full lifecycle of adding an API across backend + SDK.
|
||||
|
||||
---
|
||||
|
||||
@@ -197,7 +175,7 @@ Recipes are tested, copy-pasteable patterns for a task, one layer above the per-
|
||||
|
||||
[src/gui/](src/gui/) is the Puter desktop: deliberately plain JavaScript + jQuery with HTML-string templates. Don't introduce a UI framework or a new rendering pattern.
|
||||
|
||||
**Conformity over novelty**: match the existing design and code structure even where you'd choose differently. A visually or structurally divergent addition is a defect even when it works.
|
||||
The guiding rule here is **conformity over novelty** — match the existing design and code structure even where you'd personally choose differently. A visually or structurally divergent addition is a defect even when it works.
|
||||
|
||||
- **Reuse existing UI primitives before writing new ones.** Windows and dialogs are `UIWindow*` functions in [src/gui/src/UI/](src/gui/src/UI/); generic pieces already exist (`UIAlert`, `UIPrompt`, `UIContextMenu`, `UINotification`, `UIPopover`, widgets in [UI/Components/](src/gui/src/UI/Components/)). A new window should read like its neighbors: an async function taking an options object, composing an HTML string, wiring behavior with jQuery, delegating to `UIWindow(...)`.
|
||||
- **Match the visual language.** Use existing CSS classes (`button`, `button-primary`, window chrome, form styles) and copy the layout patterns of neighboring windows; new styles go in [src/gui/src/css/](src/gui/src/css/) following existing conventions. Verify anything positional (menus, overlays, z-index) on both desktop and mobile viewports.
|
||||
|
||||
+8
-26
@@ -17,21 +17,6 @@ Out-of-scope:
|
||||
- Denial of Service (DoS), spam, or volumetric attacks.
|
||||
- Physical security issues.
|
||||
|
||||
## Eligibility
|
||||
|
||||
We pay bounties for issues that threaten user security and privacy, or the security and health of the platform. We appreciate every report, but hardening suggestions and correctness bugs are not eligible, even when we fix them. Please open a GitHub issue for those.
|
||||
|
||||
A report must show how an attacker can directly perform the action against another account or the system. It is not eligible if it:
|
||||
|
||||
- Starts from access the attacker should not already have: a stolen session or token, the victim's password, a root or full-access token, or script execution on `puter.com`.
|
||||
- Needs the victim to do more than ordinary use (opening a link or an app), such as publishing their own secrets, sending a verification link to an address they don't control, or approving a prompt that accurately states what it grants.
|
||||
- Gives the attacker nothing they couldn't already get through a supported API.
|
||||
- Only reproduces in a non-default or development setup (SQLite, a `custom` OIDC provider, an unset outbound proxy, cookies or config planted by hand). Findings in the open-source code must affect `puter.com` or a default self-hosted install.
|
||||
|
||||
**Shared state.** Sharing files, KV entries, or cross-app KV access means the sharer trusts the sharee. Exploits that depend on that trust, such as a sharee misusing access the sharer granted, may be treated as hardening or lower severity.
|
||||
|
||||
Hardening we ship in a public PR does not make similar code paths vulnerabilities. They are judged by these same rules.
|
||||
|
||||
## Known Non-Issues (Please Check Before Submitting)
|
||||
|
||||
The following have already been reviewed and determined **not to be vulnerabilities**. Reports that only re-describe one of these are **not eligible for a reward** and will be closed as non-issues — even if they include new code references. Please review this list before submitting:
|
||||
@@ -42,12 +27,11 @@ The following have already been reviewed and determined **not to be vulnerabilit
|
||||
- **Apps invoking drivers, creating workers, or using KV.** Applications are intended to do this; worker permissions are scoped to the owning app. This is by design.
|
||||
- **App metadata or app user-count "leaks".** This information is currently public by design. This covers **private apps** too: private apps are an unreleased, admin-only beta for paid listings in the Puter App Center. Their `index_url` and other metadata are not secret, the flag is not user-settable, and no production app relies on it. Reports that a private app's metadata is reachable through a sibling route (taskbar items, `/get-launch-apps`, `index_url` drift on the hosting gate, etc.) are normalization work, not vulnerabilities.
|
||||
- **Worker hot-reload driven by a share recipient.** Sharing a worker's source file with write access is how you let a collaborator change that worker's code and have it deploy live. The redeploy running as the file owner, with the owner's worker credentials, is the intended behavior of that grant — the owner chose to hand out write access to executable code. If you do not want that, do not share the source file with write access.
|
||||
- **Re-claiming a site or worker name its owner deleted.** A hosted site or worker name that its owner deletes is treated as abandoned by that owner. If one of their apps still points at it, whoever claims the name next serves that app's `index_url` and receives what the app is launched with, and a new app on the same origin may share the old one's identity. Guarding app identities against being recycled this way is ongoing hardening, not a security boundary, so new ways to re-claim an abandoned name, or to inherit an app's tokens or data through one, are not eligible.
|
||||
- **Share-mode downgrades that do not cascade.** Demoting a delegate does not retroactively revoke the grants that delegate already issued downstream. Shares are independent grants once issued, not a live-evaluated chain. Revoking the downstream grant is a separate, supported action.
|
||||
- **Metering, credit-hold, and quota inaccuracies.** Credit holds and quota checks are up-front estimates, and actual usage is billed afterwards. Overspend that is bounded and still billed is a correctness bug, including concurrent requests that exceed a hold or quota by some margin; the per-account concurrency and rate-limit gates are the control we rely on. Report these only if you can use a service without it being billed, or overspend without bound.
|
||||
- **Existence-only oracles.** A response that tells "exists but forbidden" apart from "not found" for a path or ID the caller already supplied, without revealing anything else about it, is hardening.
|
||||
- **Concurrency / TOCTOU overspend of credits or storage quota.** Firing concurrent requests to exceed an AI credit hold or a storage quota by some margin is known and accepted; the per-account concurrency and rate-limit gates are the control we rely on, and the residual overage is bounded and billed. Report these only if you can demonstrate _unbounded_ overspend.
|
||||
- **Session and credential lifecycle around 2FA enrollment.** Enrolling in 2FA is not a credential-revocation event: existing authenticated sessions stay valid, and outstanding password-reset links stay valid. Likewise, account-management actions behind the `/user-protected/*` gate are password-gated by design, not TOTP-gated. A recovery code that stops being accepted fails _closed_ — it denies access rather than granting it, so it is a bug report, not a vulnerability.
|
||||
- **Group permissions** (`/auth/revoke-user-group` and siblings). The group-permission surface is unfinished and not exposed anywhere in the product. Findings there are not reachable by any user.
|
||||
- **Correctness bugs with no security boundary crossed.** For example, an API that reports success while silently leaving data behind. We want these reported — please open a GitHub issue — but they are not bounty-eligible.
|
||||
- **Missing Subresource Integrity (SRI) on external scripts/styles.** Accepted as hardening, not as a vulnerability, absent a demonstrated compromise of the CDN in question.
|
||||
- **General "token in a URL" / token-lifetime / token-storage designs** — signed directory URLs exposing children, a write signature implying read, app tokens outliving a web session, or the session token being held in `localStorage` on `puter.com`. These are current intended behaviors. Shorter-lived tokens with a refresh flow are on the roadmap; their absence is not a vulnerability on its own. Show a reproducible attack that does not already presuppose script execution on `puter.com`.
|
||||
- **Missing PKCE, unverified `id_token` signatures, or other OIDC hardening.** Provider tokens are obtained through a server-to-server authorization-code exchange with the provider's token endpoint over TLS, so the resulting `id_token` / userinfo claims are trusted from that channel rather than from local JWKS signature verification — the callback never accepts a caller-supplied `id_token`. Adding local signature / `aud` / `iss` / `exp` checks is welcome defense-in-depth (please open a GitHub issue/PR), but their absence is not an account-takeover vector on its own.
|
||||
@@ -68,7 +52,7 @@ To participate, you must:
|
||||
3. **Respect availability**: Do not perform denial-of-service attacks or automated scans that degrade service.
|
||||
4. **Follow disclosure policy**: Do not publicly disclose vulnerabilities until we have confirmed and patched the issue.
|
||||
5. **Act in good faith**: Make every effort to avoid privacy violations, destruction of data, and interruption or degradation of services.
|
||||
6. **Check Eligibility and Known Non-Issues**: Reports that fall outside "Eligibility" or match an item in "Known Non-Issues" above are not eligible and will be closed.
|
||||
6. **Check the Known Non-Issues list**: Reports matching an item in the "Known Non-Issues" section above are not eligible and will be closed.
|
||||
|
||||
Reports that do not meet these guidelines may not be eligible for a reward.
|
||||
|
||||
@@ -79,14 +63,14 @@ Reports **MUST** demonstrate a **working, reproducible exploit with real impact*
|
||||
- Exact steps to reproduce, the relevant request/response or code path, and the commit or version you tested.
|
||||
- The **observed** result versus the **expected** result.
|
||||
- For client-side, UI, or authentication-flow bugs: a short screen recording (≤ 2 minutes) showing the exploit working end-to-end on a real Puter instance.
|
||||
- For server-side bugs: a runnable proof-of-concept and screenshot or video showing the attack process. A unit test or in-process harness alone does not count; show it against a running instance.
|
||||
- For server-side bugs: a runnable proof-of-concept and screenshot or video showing the attack process.
|
||||
|
||||
Reports based solely on reading the source ("source review only, not tested") or unverified AI/LLM-generated reports are the **lowest triage priority and not eligible for bounty**.
|
||||
If you used an AI tool to help find an issue, you must personally verify that it actually reproduces before submitting. Reports sent by automated agents are not eligible.
|
||||
If you used an AI tool to help find an issue, you must personally verify that it actually reproduces before submitting.
|
||||
|
||||
Please submit **one issue per report**.
|
||||
Bundled "audit packs" of many speculative findings will not be eligible for bounty; send each confirmed issue separately.
|
||||
Additionally please **DO NOT** submit too many issues at once; these will be treated as a bundled report, and will not be eligible for bounty.
|
||||
Bundled "audit packs" of many speculative findings will not be elligible for bounty; send each confirmed issue separately.
|
||||
Additionally please **DO NOT** submit too many issues at once; these will be treated as a bundled report, and will not be elligible for bounty.
|
||||
|
||||
## Reporting Process
|
||||
|
||||
@@ -99,8 +83,6 @@ Include:
|
||||
- Suggested remediation (if available)
|
||||
- Paypal id where we can send the bounty (if eligible)
|
||||
|
||||
Send the full report in your first email. We don't reply to messages that only ask whether we run a bounty or where to report.
|
||||
|
||||
We aim to acknowledge receipt within **72 hours** and provide a resolution timeline.
|
||||
|
||||
## Reward Structure
|
||||
@@ -113,7 +95,7 @@ We offer monetary rewards based on the severity of the vulnerability, as determi
|
||||
- **Low: \$50 – \$100**
|
||||
|
||||
Non-security issues, suggestions, and best practices feedback are always welcome, but may not qualify for a reward.
|
||||
If multiple researchers report the same issue, the bounty will be awarded to the first eligible report we receive. Issues we already know about, from internal review, an open ticket, or a fix already in a public commit or PR, count as duplicates.
|
||||
If multiple researchers report the same issue, the bounty will be awarded to the first eligible report we receive.
|
||||
|
||||
## Payments Disclaimer
|
||||
|
||||
|
||||
+36
-11
@@ -1,21 +1,46 @@
|
||||
# Contributing to Puter
|
||||
# Contributing to Puter Backend
|
||||
|
||||
Thanks for contributing. These rules aren't strictly enforced, but following them makes every PR easier. If anything's unclear, ping a core maintainer or open the PR and ask.
|
||||
Thanks for contributing. These rules aren't strictly enforced — but following them makes every PR easier. If anything's unclear, ping a core maintainer or open the PR and ask.
|
||||
|
||||
- New to the backend? Start with [doc/architecture.md](doc/architecture.md).
|
||||
- Code, test, and docs conventions for the whole repo live in [AGENTS.md](AGENTS.md). It's written for AI agents; the conventions apply to human contributors too.
|
||||
- Adding or changing a public API (an endpoint, driver method, or puter.js method)? Follow [doc/contributing-apis.md](doc/contributing-apis.md): backward compatibility, [developer docs](src/docs/), types, and tests all move in the same PR.
|
||||
- Found a vulnerability? Report it privately per [SECURITY.md](SECURITY.md).
|
||||
New to the backend? Start with [doc/architecture.md](doc/architecture.md).
|
||||
|
||||
---
|
||||
|
||||
## Before you open a PR
|
||||
## 1. Test it. Run it.
|
||||
|
||||
Check your diff against these sections of AGENTS.md:
|
||||
Run the affected code path end-to-end before opening a PR. "It builds" is not "it works."
|
||||
|
||||
- [Working rules of thumb](AGENTS.md#working-rules-of-thumb): run it end to end, test new behavior and bug fixes, understand every line you commit (AI assistance is fine), leave touched files a bit better, and publish limit changes.
|
||||
- [Reuse and abstraction](AGENTS.md#reuse-and-abstraction) and [hot paths](AGENTS.md#hot-paths).
|
||||
- [Security & privacy](AGENTS.md#security--privacy): expose nothing the caller doesn't need.
|
||||
Add tests for new behavior, endpoints, or bug fixes. If something's genuinely hard to test, say so in the PR.
|
||||
|
||||
## 2. Follow existing patterns
|
||||
|
||||
Match the shape of similar code already in the repo. [doc/architecture.md](doc/architecture.md) is the source of truth for layers, wiring, and naming. If you think a pattern is wrong, raise it — don't quietly diverge.
|
||||
|
||||
In plain-JS files, typing is encouraged via JSDoc `@type` annotations using the TypeScript type system, with `@typedef` for shared shapes. Don't type API surfaces as `unknown` or untyped `...args` unless the values are passed through transparently to an upstream layer that owns their type.
|
||||
|
||||
## 3. Don't expose system or user information
|
||||
|
||||
Scan your diff for stray logs, debug routes, internal paths, secrets, tokens, or user data in errors/responses. When in doubt, return less. Flag any auth, permission, or data-export changes in the PR description.
|
||||
|
||||
For private security reports, see [SECURITY.md](SECURITY.md).
|
||||
|
||||
## 4. AI-assisted code is fine — understood code is required
|
||||
|
||||
Don't commit code you couldn't have written, debugged, or defended yourself. Read the diff, run it, and be ready to explain it in review.
|
||||
|
||||
## 5. Adding or changing APIs
|
||||
|
||||
If you add or change a public API (an endpoint, driver method, or puter-js method), follow [doc/contributing-apis.md](doc/contributing-apis.md) — backward compatibility, [developer docs](src/docs/), types, and tests all move in the same PR.
|
||||
|
||||
The same rule applies to limits: rate limits, concurrency caps, quotas, and allowances are published in [src/docs/src/rate-limits-and-quotas.md](src/docs/src/rate-limits-and-quotas.md), and a PR that changes one of those numbers updates that page in the same PR. A limit nobody published is a limit developers only discover as a service failure.
|
||||
|
||||
## 6. Boy Scout Rule — leave it 1% better
|
||||
|
||||
|
||||
|
||||
Fix the typo, the dead import, the missing test, the bit you had to read twice. Keep cleanup proportional to the change — no refactors riding along on bug fixes.
|
||||
|
||||
---
|
||||
|
||||
## Opening a PR
|
||||
|
||||
|
||||
+2
-4
@@ -14,7 +14,7 @@
|
||||
# config.default.json, so partial overrides work. Absent file = defaults.
|
||||
|
||||
# ---- Build stage ----
|
||||
FROM node:26-bookworm-slim AS build
|
||||
FROM node:24-slim AS build
|
||||
|
||||
WORKDIR /opt/puter
|
||||
|
||||
@@ -59,7 +59,7 @@ RUN set -e; \
|
||||
wait $pjs_pid
|
||||
|
||||
# ---- Runtime stage (slim — no build tools) ----
|
||||
FROM node:26-bookworm-slim
|
||||
FROM node:24-slim
|
||||
|
||||
WORKDIR /opt/puter
|
||||
|
||||
@@ -79,8 +79,6 @@ ENV PUTER_CONFIG_PATH=/etc/puter/config.json
|
||||
ENV NODE_OPTIONS=--enable-source-maps
|
||||
|
||||
EXPOSE 4100
|
||||
# The inbound SMTP receiver, when run from this image with the `smtp` profile.
|
||||
EXPOSE 2525
|
||||
|
||||
USER node
|
||||
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@
|
||||
"disable_user_signup": false,
|
||||
"teams_enabled": false,
|
||||
"max_teams_per_user": 1,
|
||||
"max_seats_per_team_free": 3,
|
||||
"max_seats_per_team_free": 4,
|
||||
"max_seats_per_team_paid": 40,
|
||||
"strict_email_verification_required": false,
|
||||
"gui_assets_root": "./src/gui",
|
||||
|
||||
+12
-55
@@ -7,8 +7,9 @@
|
||||
// Each setting lives at exactly one canonical key — there are no fallback
|
||||
// aliases. Values shown are illustrative, not production secrets.
|
||||
//
|
||||
// Keys consumed only by closed-source / hosted-prod extensions are
|
||||
// intentionally omitted.
|
||||
// Keys consumed only by closed-source / hosted-prod extensions (clickhouse,
|
||||
// cacheUpdateHandler, pages, prodMeteringAndBilling, …) are intentionally
|
||||
// omitted.
|
||||
|
||||
// ── Environment / identity ──────────────────────────────────────────
|
||||
"config_name": "template",
|
||||
@@ -34,7 +35,7 @@
|
||||
// Idle keep-alive timeout in ms. Must stay above the idle timeout of any
|
||||
// proxy in front, or the proxy reuses connections this server has closed
|
||||
// and its clients see 502s.
|
||||
"keep_alive_timeout": 920000,
|
||||
"keep_alive_timeout": 620000,
|
||||
"protocol": "http",
|
||||
"domain": "puter.localhost",
|
||||
// Fully-qualified externally-visible URL. Computed from protocol/domain/
|
||||
@@ -215,9 +216,7 @@
|
||||
// "bootstrapTables": true,
|
||||
"path": "",
|
||||
// Credentials. NOTE: snake_case here, unlike `s3.s3Config` below.
|
||||
// For dynamodb-local, any non-empty values work. On AWS,
|
||||
// `"useCredentialChain": true` uses the default credential chain
|
||||
// instead of keys.
|
||||
// For dynamodb-local, any non-empty values work.
|
||||
"aws": {
|
||||
"access_key": "",
|
||||
"secret_key": "",
|
||||
@@ -256,38 +255,6 @@
|
||||
}
|
||||
},
|
||||
|
||||
// ── Inbound mail (separate process) ─────────────────────────────────
|
||||
// Mailboxes addressed at one of your own domains. `secret` gates the
|
||||
// ingress endpoint that files a message into the recipient's ~/.mail.
|
||||
//
|
||||
// `localServer` additionally runs an SMTP listener that accepts mail from
|
||||
// other mail servers and hands each message to that endpoint. It runs as
|
||||
// its own process - `npm run start:smtp`, or the `smtp` compose profile -
|
||||
// and stores nothing itself. Without it, nothing delivers to the endpoint
|
||||
// unless you point something else at it.
|
||||
//
|
||||
// The listener is unauthenticated by nature: anything on the internet may
|
||||
// hand it a message for an address in `localDomains`. It performs no spam,
|
||||
// SPF, DKIM or DMARC checks. Read the self-hosting guide before exposing
|
||||
// it.
|
||||
// "userEmail": {
|
||||
// "secret": "a long random string",
|
||||
// "localServer": true,
|
||||
// "localDomains": ["example.com"],
|
||||
// // Container-internal by default; publish 25 onto it from outside.
|
||||
// "localPort": 2525,
|
||||
// // Defaults to <api_base_url>/email/ingress. The endpoint is served
|
||||
// // on the `api` subdomain, so when this points somewhere internal,
|
||||
// // `localIngressHost` has to name that virtual host.
|
||||
// "localIngressUrl": "http://puter:4100/email/ingress",
|
||||
// "localIngressHost": "api.example.com",
|
||||
// "localHostname": "mx.example.com",
|
||||
// "localMaxRecipients": 50,
|
||||
// // Concurrent connections. Each message is held in memory while it
|
||||
// // is delivered, so this also bounds memory use.
|
||||
// "localMaxClients": 20
|
||||
// },
|
||||
|
||||
// ── Sharing ─────────────────────────────────────────────────────────
|
||||
// Email a recipient who already has an account about a new share. On
|
||||
// unless set to false; they opt out via the unsubscribe link, or by
|
||||
@@ -315,8 +282,7 @@
|
||||
// The tables ship either way and sit inert; this gates whether `/teams`
|
||||
// is registered at all, so with it off the paths do not exist rather than
|
||||
// existing and refusing. It is also the backout: turning it off removes
|
||||
// the feature without touching data. Teams ships as an extension; on,
|
||||
// the server refuses to boot unless one registers it.
|
||||
// the feature without touching data.
|
||||
"teams_enabled": false,
|
||||
//
|
||||
// Live teams one user may own. Soft-deleted ones do not count, so
|
||||
@@ -325,13 +291,12 @@
|
||||
//
|
||||
// Seats one team may provision. Each seat is a real Puter account taking a
|
||||
// name from the global username pool, so this is what bounds a team's blast
|
||||
// radius. The owner's plan decides which of the two applies; defaults are 3
|
||||
// and 40. A plan registered with `teamSeatCap` uses that instead. Lowering
|
||||
// a cap below a team's current seat count blocks new provisioning and
|
||||
// disables nobody. Deployments upgrading from the old flat default of 50:
|
||||
// without a billing extension every owner reads as free, so set
|
||||
// `max_seats_per_team` to keep the old behavior.
|
||||
"max_seats_per_team_free": 3,
|
||||
// radius. The owner's plan decides which of the two applies; defaults are 4
|
||||
// and 40. Lowering either below a team's current seat count blocks new
|
||||
// provisioning and disables nobody. Deployments upgrading from the old flat
|
||||
// default of 50: without a billing extension every owner reads as free, so
|
||||
// set `max_seats_per_team` to keep the old behavior.
|
||||
"max_seats_per_team_free": 4,
|
||||
"max_seats_per_team_paid": 40,
|
||||
//
|
||||
// One flat cap for every team, whatever the owner pays. Setting it
|
||||
@@ -433,8 +398,6 @@
|
||||
"mistral": { "apiKey": "" },
|
||||
"xai": { "apiKey": "" },
|
||||
"moonshot": { "apiKey": "" },
|
||||
// MiniMax. One key powers chat and video generation (H3, Hailuo);
|
||||
// video calls go to the same host, beside the chat API's `/v1`.
|
||||
"minimax": {
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://api.minimax.io/v1"
|
||||
@@ -455,10 +418,6 @@
|
||||
"apiKey": "",
|
||||
"apiBaseUrl": "https://ark.ap-southeast.bytepluses.com/api/v3"
|
||||
},
|
||||
// Kling AI. One API key (not the legacy AccessKey/SecretKey pair)
|
||||
// powers video generation, image generation and TTS; `apiBaseUrl`
|
||||
// defaults to https://api-singapore.klingai.com.
|
||||
"kling": { "apiKey": "" },
|
||||
"zai": { "apiKey": "" },
|
||||
"alibaba": { "apiKey": "" },
|
||||
"hoonify": {
|
||||
@@ -497,8 +456,6 @@
|
||||
"defaultVoiceId": "",
|
||||
"speechToSpeechModelId": ""
|
||||
},
|
||||
// Both AWS providers also take `"useCredentialChain": true` in place
|
||||
// of keys, like `dynamo.aws`.
|
||||
"aws-polly": {
|
||||
"access_key": "",
|
||||
"secret_key": "",
|
||||
|
||||
+3
-9
@@ -36,10 +36,9 @@ pass one explicitly unless you really mean "page someone".
|
||||
### Choosing one
|
||||
|
||||
- Did the server fail to do its job in a way nobody expected? → `critical`
|
||||
- Is a background job, rate, or dependency degraded, or has an account spent
|
||||
far past its credits? → `warning`
|
||||
- Is this a user doing something notable (tripping an abuse heuristic)?
|
||||
→ `info`
|
||||
- Is a background job, rate, or dependency degraded? → `warning`
|
||||
- Is this a user doing something notable (tripping an abuse heuristic,
|
||||
overspending)? → `info`
|
||||
- Did one of *our own* limits reject a caller — a rate limit, a concurrency
|
||||
cap, a quota? → don't alarm at all. The limit doing its job is not an
|
||||
event; the 429 is the whole signal, and alarming on it only produces noise
|
||||
@@ -134,11 +133,6 @@ occurrence is reported up to a small burst, then one per interval. A fault
|
||||
recurring hundreds of times a second still counts every occurrence, but it
|
||||
does not write a log line or build an alert payload for each one.
|
||||
|
||||
The count lasts only as long as the client remembers the alarm. One that
|
||||
hasn't recurred for 24 hours is forgotten, and the client holds at most 1000
|
||||
at once, dropping the least recently raised first. Either way the next
|
||||
occurrence is a new alarm, with its count and burst starting over.
|
||||
|
||||
Below that, the chat transport won't repost the same alarm id within
|
||||
`repeatThrottleMs` (default 15 minutes). The first occurrence always posts,
|
||||
and the next one that gets through reports how many piled up in between —
|
||||
|
||||
+20
-9
@@ -1,6 +1,6 @@
|
||||
# Backend Architecture
|
||||
|
||||
Loosely inspired by the Controller–Service–Repository pattern with dependency injection. The backend is a stack of layers; each depends only on the layers beneath it and receives them through its constructor from `PuterServer` ([src/backend/server.ts](../src/backend/server.ts)), which instantiates the layers in order. The rules for working within the layers are in [AGENTS.md](../AGENTS.md#backend).
|
||||
Loosely inspired by the Controller–Service–Repository pattern with dependency injection. The backend is organized as a stack of layers where each layer only depends on the layers beneath it, and `PuterServer` ([src/backend/server.ts](../src/backend/server.ts)) instantiates each layer in order and hands the instances down to the next.
|
||||
|
||||
## Layers
|
||||
|
||||
@@ -31,6 +31,8 @@ block-beta
|
||||
style CFG fill:#334155,stroke:#1e293b,color:#fff
|
||||
```
|
||||
|
||||
Each layer only depends on the layers beneath it, and every dependency is injected through the constructor by `PuterServer`. Extensions sit alongside this stack and can register into any layer — see [Extensions](#extensions) below.
|
||||
|
||||
| Layer | Lives in | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| **Controllers** | [src/backend/controllers/](../src/backend/controllers/) | Route handlers. Parse + validate input, apply per-route gates (auth, subdomain, rate limit, body parsers — see `RouteOptions`), call into services, format responses. |
|
||||
@@ -40,7 +42,7 @@ block-beta
|
||||
| **Clients** | [src/backend/clients/](../src/backend/clients/) | Adapters for external/internal services (sql, redis, s3, dynamodb, email, event bus, …). Knows protocols, not domain concepts. |
|
||||
| **Config** | `config.*.json` → `IConfig` | The flat, typed config object every layer receives at construction. |
|
||||
|
||||
Extensions sit alongside this stack and can register into any layer; see [Extensions](#extensions).
|
||||
Each layer receives the layers beneath it through its constructor, so dependencies are explicit and traceable from `PuterServer`. A controller does not reach into a client directly; if it needs one, the right move is usually a service.
|
||||
|
||||
## Entry point: `PuterServer`
|
||||
|
||||
@@ -53,11 +55,13 @@ Extensions sit alongside this stack and can register into any layer; see [Extens
|
||||
|
||||
## Context (ALS)
|
||||
|
||||
[`Context`](../src/backend/core/context.ts), backed by `AsyncLocalStorage`, carries per-request state without threading it through every function signature. The request-context middleware opens a scope per request after the auth probe runs, so anything inside a request handler can call `Context.get('actor')` / `Context.get('req')`. It's used sparingly, mostly for `actor` and `req`; see the [Context rule](../AGENTS.md#backend).
|
||||
We use [`Context`](../src/backend/core/context.ts) — backed by `AsyncLocalStorage` — to carry per-request state without threading it through every function signature. It is used **sparingly**, mostly for `actor` and `req`. The request-context middleware opens a scope per request after the auth probe runs; anything inside a request handler can call `Context.get('actor')` / `Context.get('req')` instead of plumbing it as an argument.
|
||||
|
||||
Prefer explicit arguments. Reach for `Context` only when the value is truly request-scoped and would otherwise need to thread through many layers.
|
||||
|
||||
## Actors
|
||||
|
||||
An [`Actor`](../src/backend/core/actor.ts) is who a request is acting as. Its two app fields are not interchangeable: `app` is the app the actor carries *directly* (an app-under-user token), while `effectiveApp` is the app it ultimately acts as — its own, or the one that issued its access token. An access-token actor has no `app` of its own, so a gate that reads `app` answers "no app" for it and falls open.
|
||||
An [`Actor`](../src/backend/core/actor.ts) is who a request is acting as. It has two app-shaped fields and they are not interchangeable: `app` is the app the actor carries *directly* (an app-under-user token), while `effectiveApp` is the app it ultimately acts as — its own, or the one that issued its access token. An access-token actor has no `app` of its own, so a gate that reads `app` answers "no app" for it and falls open.
|
||||
|
||||
`makeActor` resolves `effectiveApp` once at construction, and `assertResolvedActor` at the request edge rejects any actor that skipped it — so reading `effectiveApp` needs no fallback. Reach for `app` only where the direct app genuinely is the question, and say why in a comment.
|
||||
|
||||
@@ -67,11 +71,11 @@ An [`Actor`](../src/backend/core/actor.ts) is who a request is acting as. Its tw
|
||||
|
||||
## Extensions
|
||||
|
||||
Extensions live in [extensions/](../extensions/), parallel the layered stack, and hold the parts of the system Puter still works without. [AGENTS.md](../AGENTS.md#extensions) has the rule for what belongs in one.
|
||||
Extensions live alongside core ([packages/puter/extensions/](../extensions/)) and parallel the layered stack. They are meant for **non-crucial parts of the system** — things Puter still works without if removed.
|
||||
|
||||
- **Good extensions**: [thumbnails](../extensions/thumbnails.ts), [serverInfo](../extensions/serverInfo.ts), [devWatcher](../extensions/devWatcher.ts) — opt-in features cleanly bolted on.
|
||||
- **Should probably be core**: [metering](../extensions/metering.ts), [appTelemetry](../extensions/appTelemetry.ts) — clients now expect these to be present, so the "extension" framing is misleading.
|
||||
- **Shouldn't have been an extension**: [whoami](../extensions/whoami.ts) — it's load-bearing for every authenticated client.
|
||||
- **Shouldn't have been an extension**: [whoami](../extensions/whoami.ts) — it's load-bearing for every authenticated client. Keep this one in mind as a cautionary example when deciding whether something belongs in an extension.
|
||||
|
||||
### Extension API
|
||||
|
||||
@@ -94,11 +98,11 @@ import { extension } from '@heyputer/backend/src/extensions';
|
||||
const services = extension.import('service');
|
||||
|
||||
extension.get('/healthcheck/deep', { subdomain: 'api', adminOnly: true }, async (_req, res) => {
|
||||
res.json(await services.health.getStatus());
|
||||
res.json({ ok: await services.health.runDeepCheck() });
|
||||
});
|
||||
|
||||
extension.on('puter.signup.success', (_key, data) => {
|
||||
console.log('new user', data.username);
|
||||
extension.on('user.signup', (_key, data) => {
|
||||
console.log('new user', data.user.username);
|
||||
});
|
||||
```
|
||||
|
||||
@@ -113,3 +117,10 @@ extension.on('app.recommended', (_key, data) => {
|
||||
data.appNames = ['editor', 'camera'];
|
||||
});
|
||||
```
|
||||
|
||||
## Conventions
|
||||
|
||||
- **TypeScript preferred** in new code where feasible. Existing JS is fine; convert opportunistically when you're already touching a file.
|
||||
- **`camelCase`** for variable/function names; **`PascalCase`** for classes and for files that contain a class (`AuthService.ts`, `KVStoreDriver.ts`).
|
||||
- **Deduplicate**. If two services need the same logic, lift it into a util/helper rather than calling sideways across the same layer — services should not depend on other services for code reuse.
|
||||
- **Don't reach across layers.** Controllers do not poke clients directly; services do not register routes. If you find yourself wanting to, that's usually a signal the abstraction is wrong.
|
||||
+29
-20
@@ -12,7 +12,21 @@ Every change is backward compatible unless a maintainer has explicitly agreed to
|
||||
|
||||
## Core or extension?
|
||||
|
||||
The first decision is where the API lives. If **nothing in core will call it, prefer an extension** over wiring it into core. [AGENTS.md](../AGENTS.md#extensions) has the rule and its test; [architecture.md](architecture.md#extension-api) documents the `extension` API.
|
||||
The first decision is where the API lives.
|
||||
|
||||
If the API is **not crucial to core functionality — nothing in core will call it — prefer an extension** over wiring it into core. Extensions live in [extensions/](../extensions/), parallel the core layered stack, and reach core through `extension.import(...)`:
|
||||
|
||||
```js
|
||||
const services = extension.import('service');
|
||||
const stores = extension.import('store');
|
||||
|
||||
extension.registerDriver('myFeature', MyFeatureDriver); // first-class driver
|
||||
extension.post('/my-feature/frobnicate', opts, handler); // or plain routes
|
||||
```
|
||||
|
||||
Follow the same layered structure inside the extension (driver/controller → service → store) unless it genuinely only needs a couple of route handlers — then the lightweight `extension.get/post/...` helpers are enough on their own.
|
||||
|
||||
The test is the direction of dependency: Puter must still work with the extension removed. The moment core needs to call your API, it belongs in core — see [whoami](../extensions/whoami.ts) for the cautionary example of a load-bearing "extension".
|
||||
|
||||
## Adding a new API
|
||||
|
||||
@@ -21,13 +35,13 @@ Work through all seven steps; the PR is complete when every one is.
|
||||
### 1. Design the surface first
|
||||
|
||||
- Sketch the signature, options, return shape, and error cases before writing code. Find the two or three most similar existing APIs and match their conventions.
|
||||
- New parameter and field names follow the [naming rule](../AGENTS.md#language--files) (`camelCase`).
|
||||
- New parameter and field names are `camelCase`. (Existing `snake_case` names stay where they already exist.)
|
||||
- Anything returning a list follows the [pagination convention](pagination.md): `limit`/`cursor` in, `{ items, cursor, total }` envelope out.
|
||||
- Prefer an options object over a growing list of positional parameters, but keep the common case callable with a single argument where siblings do.
|
||||
|
||||
### 2. Backend
|
||||
|
||||
Follow the layered stack ([architecture.md](architecture.md), [layer rules](../AGENTS.md#backend)): a controller or driver at the edge, business logic in a service, persistence in a store. Return exactly what the caller needs and no more — every response field you ship is permanent — and use stable `snake_case` error codes.
|
||||
Follow the layered stack ([architecture.md](architecture.md)): a controller or driver at the edge, business logic in a service, persistence in a store. The edge parses and validates input and applies gates; services assume the caller is already authorized. Return exactly what the caller needs and no more — every response field you ship is permanent — and use stable `snake_case` error codes.
|
||||
|
||||
#### Controller or driver?
|
||||
|
||||
@@ -40,16 +54,10 @@ Both are supported ways to define an API. **Prefer a controller when you need fi
|
||||
|
||||
- **Controller routes** (and extension routes — same options) take [`RouteOptions`](../src/backend/core/http/types.ts): auth gates (`requireAuth`, `requireUserActor`, `noUserSession`, `adminOnly`, `allowedAppIds`, and the access-token controls), `subdomain` routing, per-route `rateLimit`, body parsers, and arbitrary extra `middleware`. The auth flavors are subtle and default-deny — read the JSDoc on each field before picking.
|
||||
- **Driver methods** get their policies from the `@Driver` options: per-method `rateLimit` (limit/window/backend), `concurrent` in-flight caps (optionally `bySubscription`), `requireSubscription`, `requireReputation`, and `noUserSession`. The `/drivers/call` surface enforces them.
|
||||
- **Paying accounts only:** `requireSubscription`, as a route option on a controller endpoint or in the per-method `@Driver({ requireSubscription })` block on a driver (`/drivers/call` is one shared route, so a driver's requirement can't live in route options). `true` accepts any plan that isn't free, so a plan an extension registers counts without core naming it; an array of policy ids (`['business', 'pro']`) accepts only those; `false` or leaving it out means no requirement. An empty array is a boot error, since it would read as subscribers-only while admitting everyone. The check is a map lookup on the metering service's cached per-actor subscription. It asks which plan the account is on; `requireCredits` asks whether it has budget left. Deployments with no paid plans (self-hosted installs) turn it off with `meteringEnforcement.subscriptions: false`.
|
||||
- **Trusted-enough accounts only:** `requireReputation`, as a route option or in the per-method `@Driver({ requireReputation })` block. Opt-in; implies `requireAuth`. The value names a tier, and `reputationGate.tiers` in config sets the score that tier takes, so a deployment can retune it without editing the surface. A tier the running config doesn't define is inert and everyone passes, so an install that doesn't score its accounts never turns traffic away on a score it never computed. `reputationGate.enabled: false` stops every declared gate at once. Denials are a bare 403 `reputation_required` that names neither the score nor the tier. Nothing in the tree declares one yet.
|
||||
- **A specific verified factor:** `requireVerified` (email, and only under `strict_email_verification_required`), `requirePhoneVerified`, `requireCardVerified`. Opt-in, and the factor must actually have been verified, unlike the default-on gate that turns away accounts still carrying a pending verification the abuse harness asked for. `requireAnyVerified: ['phone', 'card']` is the OR of the last two: any listed factor verified at any point passes, as does a paid plan when `card` is listed (a paying account already has a card on file). Otherwise only the factors the deployment can verify are asked for (an SMS provider configured, a card gate an extension reports on); with none verifiable the gate is inert rather than locking the route on a self-hosted install. A denial carries the first verifiable factor's code plus `factors` (the verifiable ones, in the route's order), so a client can lead with one flow and offer the other. `verifiedFactorGate.enabled: false` in config stops every declared gate at once, e.g. during an SMS-provider outage.
|
||||
- **Metered spend:** an endpoint that spends metered resources on the caller's behalf (moving file content, object-store requests, anything else the account is billed for) declares `requireCredits: true`, which turns an account with no budget left away with a 402 before the handler runs. Endpoints that only describe or delete things deliberately don't: an account that has run out still has to see what it has, clear it, and reach its billing pages. Drivers have no route options, so they call `assertActorHasCredits` themselves ([src/backend/services/metering/enforcement.ts](../src/backend/services/metering/enforcement.ts)) — see `KVStoreDriver`, which does it once for every method.
|
||||
|
||||
Provider sign-in requires the account's authenticator or a recovery code when 2FA is enabled, including signup callbacks that resolve an existing account. The five-minute challenge uses the same single-use `/login/otp` and `/login/recovery-code` verification as password sign-in. The GUI carries it in a URL fragment and removes it before initialization. A session and successful popup-return proof are issued only after verification; cancelling retains the popup's signed retry context without marking it authenticated.
|
||||
|
||||
Disabling 2FA through `/user-protected/disable-2fa` requires both password or OIDC revalidation and the current six-digit authenticator `code`. Missing, malformed, or incorrect codes leave the secret and recovery codes intact. A team's requirement for 2FA still prevents disabling it.
|
||||
|
||||
Email changes through `/user-protected/change-email` require the password or OIDC revalidation cookie, plus a six-digit `code` from the authenticator when 2FA is enabled. Missing or malformed codes return `400 bad_request`; incorrect codes return `401 code_mismatch`. Both include `factor: 'otp'`. Confirmation links are bound to that verified request. A 2FA-enabled account must request a new link if its existing link was issued without second-factor verification.
|
||||
- **A surface only paying accounts should reach** declares `requireSubscription` — the route option on a controller endpoint, the per-method `@Driver({ requireSubscription })` block on a driver (`/drivers/call` is one shared route, so a driver's requirement can't live in route options). `true` accepts any plan that isn't free, so a plan registered by an extension counts without core naming it; an array of policy ids (`['business', 'pro']`) accepts only those; `false` is "no requirement", the same as leaving it out. An empty array is a boot error rather than a silent no-op — it reads as subscribers-only while admitting everyone. Off unless asked for, and answered from the metering service's cached per-actor subscription, so it costs a map lookup. Distinct from `requireCredits`: this asks which plan the account is on, not whether it has budget left. Deployments with no paid plans (self-hosted installs) turn the whole thing off with `meteringEnforcement.subscriptions: false`.
|
||||
- **A surface only a trusted-enough account should reach** declares `requireReputation` — the route option on a controller endpoint, the per-method `@Driver({ requireReputation })` block on a driver. The value names a tier; what that tier takes is `reputationGate.tiers` in config, so the score a surface is worth is a deployment call and can be retuned without editing the surface. A tier the running config doesn't define is inert and everyone passes — an install that doesn't score its accounts must not start turning traffic away on a score it never computed — and `reputationGate.enabled: false` stops every declared gate at once. Opt-in, implies `requireAuth`, and denies with a bare 403 `reputation_required` that names neither the score nor the tier. Nothing in the tree declares one yet: the mechanism ships ahead of the enrollment.
|
||||
- **An account must have verified a specific factor** — `requireVerified` (email, and only under `strict_email_verification_required`), `requirePhoneVerified`, `requireCardVerified`. These are opt-in and require the factor to have actually been verified, unlike the default-on gate that turns away accounts still carrying a pending verification the abuse harness asked for. `requireAnyVerified: ['phone', 'card']` is the OR of the last two: any listed factor verified at any point passes, as does a paid plan where `card` is listed (a paying account has a card on file already); otherwise only the factors the deployment can verify are asked for (an SMS provider configured, a card gate an extension reports on), and with none verifiable the gate is inert rather than locking the route on a self-hosted install. It denies with the first verifiable factor's code plus `factors`, the verifiable ones in the route's order, so a client can lead with one flow and offer the other. `verifiedFactorGate.enabled: false` in config stops every declared gate at once, for an SMS-provider outage.
|
||||
- **An endpoint that spends metered resources** on the caller's behalf — moving file content, making object-store requests, anything else the account is billed for — also declares `requireCredits: true`, which turns an account with nothing left of its budget away with a 402 before the handler runs. Endpoints that only describe or delete things deliberately don't: an account that has run out still has to be able to see what it has, clear it, and reach its billing pages. Drivers have no route options to declare this on, so they call `assertActorHasCredits` themselves ([src/backend/services/metering/enforcement.ts](../src/backend/services/metering/enforcement.ts)) — see `KVStoreDriver`, which does it once for every method.
|
||||
|
||||
### 3. puter.js
|
||||
|
||||
@@ -58,32 +66,33 @@ Email changes through `/user-protected/change-email` require the password or OID
|
||||
|
||||
### 4. Types
|
||||
|
||||
Type the method in JSDoc on the implementation, following the [puter.js type rules](../AGENTS.md#types) (overloads, where shapes live, `index.d.ts`), then run `npm run check:puterjs:types`. Declarations are generated from the JSDoc, so there is nothing to keep in sync by hand.
|
||||
- Type the method where you wrote it, in JSDoc: `@param`/`@returns` on the implementation, one `@overload` block per accepted call form, and `@typedef {Object}` + `@property` for any new shape. The JSDoc is the source of truth — declarations are generated from it, so there is nothing to keep in sync by hand.
|
||||
- Put a shape more than one file needs in the module's `types.js` (e.g. [src/puter-js/src/modules/kv/types.js](../src/puter-js/src/modules/kv/types.js)); anything shared across modules goes in [src/puter-js/src/lib/types.js](../src/puter-js/src/lib/types.js). A shape with one consumer can stay next to it.
|
||||
- Run `npm run check:puterjs:types` — it generates the declarations and type-checks the published surface without `skipLibCheck`. **Never edit anything under `src/puter-js/types/`**: it is gitignored build output, produced by the SDK build and shipped in the npm tarball, and the next build overwrites it.
|
||||
- Name the new type in [src/puter-js/index.d.ts](../src/puter-js/index.d.ts) if consumers should be able to import it. That file is the one hand-written declaration in the package: it decides what is public and re-exports nothing else.
|
||||
|
||||
### 5. Docs
|
||||
|
||||
- Add the method page at [src/docs/src/](../src/docs/src/)`<Area>/<method>.md` — frontmatter (`title`, `description`, `platforms`), syntax, parameters, return value, and at least one runnable example — and update the area overview (`<Area>.md`). Copy the structure of an existing page.
|
||||
- The docs are the contract users code against: signatures, defaults, and return shapes match the implementation exactly.
|
||||
|
||||
### 6. Tests
|
||||
|
||||
- **Backend:** see [backend tests](../AGENTS.md#backend-tests).
|
||||
- **SDK:** add cases to [src/puter-js/tests/api/suites/](../src/puter-js/tests/api/suites/)`<area>.suite.ts` (register new suites in `suites/index.ts`; there is no globbing). One suite runs on node, browser, and workerd via `npm run test:puterjs`; never write per-platform tests. The runners execute the built bundle, so run `npm run build:workerLib` first or the suite silently tests stale code.
|
||||
- **Backend:** colocated Vitest tests; prefer the in-memory test server (`setupPuterTestEnv` in [src/backend/testUtil.ts](../src/backend/testUtil.ts)) over mocking.
|
||||
- **SDK:** add cases to [src/puter-js/tests/api/suites/](../src/puter-js/tests/api/suites/)`<area>.suite.ts` (register new suites in `suites/index.ts`). One suite runs on node, browser, and workerd via `npm run test:puterjs` — never write per-platform tests, and rebuild first with `npm run build:workerLib` since the runners execute the built bundle.
|
||||
- **Desktop-rendered UI** (`puter.ui.*`): add a Playwright spec per [src/puter-js/TESTING.md](../src/puter-js/TESTING.md).
|
||||
|
||||
### 7. Security pass
|
||||
|
||||
Run the [security & privacy](../AGENTS.md#security--privacy) check on the diff, and confirm the auth gates are present.
|
||||
Scan the diff before opening the PR: no internals leaked in errors or logs, no over-broad responses, auth gates present. Flag anything auth-, permission-, or data-export-related in the PR description.
|
||||
|
||||
## Maintaining an existing API
|
||||
|
||||
Changes are **additive by default**:
|
||||
|
||||
- Existing call signatures keep working, including both positional and options-object forms where a method supports them. Say in the PR how existing callers are unaffected.
|
||||
- New parameters are optional, with defaults that reproduce the old behavior exactly.
|
||||
- Never rename, repurpose, or remove existing parameters, response fields, or error codes. Don't change types, ordering guarantees, or which fields are present when.
|
||||
- New behavior that could surprise existing callers goes behind an opt-in flag.
|
||||
- Docs, types, and tests move in the same PR as the behavior; a signature change with stale docs is a bug.
|
||||
- Docs, types, and tests move in the same PR as the behavior. A signature change with stale docs is a bug — the docs are the contract users code against.
|
||||
- Bug fixes come with a regression test that fails before the fix. Be suspicious of fixes that change observable behavior: someone may depend on the bug. When in doubt, ask a maintainer.
|
||||
|
||||
### Breaking changes
|
||||
|
||||
+7
-11
@@ -28,6 +28,9 @@ Paginated responses are an envelope:
|
||||
`cursor` is still returned. Never use `items.length < limit` as an
|
||||
end-of-list signal.
|
||||
|
||||
New request params are camelCase (`includeTotal`, `fetchUntilFull`).
|
||||
Pre-existing snake_case params stay for compatibility.
|
||||
|
||||
## Backward compatibility
|
||||
|
||||
Requests without pagination params keep returning the full result in the
|
||||
@@ -45,16 +48,10 @@ The envelope trigger depends on the endpoint's history:
|
||||
## Cursors
|
||||
|
||||
Cursors are opaque base64-encoded JSON, produced and consumed only by the
|
||||
backend (`src/backend/util/pagination.ts`). A cursor over a global sequence
|
||||
(an auto-increment `id`) is sealed with `sealCursor`/`openCursor` instead, so
|
||||
two pages can't be decoded into how fast the table grows; `openCursor` still
|
||||
reads a plain cursor. What a cursor wraps is an implementation detail per
|
||||
store:
|
||||
backend (`src/backend/util/pagination.ts`). What a cursor wraps is an
|
||||
implementation detail per store:
|
||||
|
||||
- DynamoDB-backed lists wrap `LastEvaluatedKey`. A kv listing of another
|
||||
namespace (cross-app or through a share handle) resumes after the last entry
|
||||
it returned instead, since the scan can stop on a private one, and its cursor
|
||||
is sealed to that listing and padded to one length.
|
||||
- DynamoDB-backed lists wrap `LastEvaluatedKey`.
|
||||
- SQL-backed lists wrap a keyset position — `(sortValue, id)` of the last
|
||||
row — and the query seeks past it (`WHERE (col, id) > (?, ?) ORDER BY col,
|
||||
id`). Any SQL list gaining a cursor must have a deterministic `ORDER BY`
|
||||
@@ -94,8 +91,7 @@ hit, the response simply carries a cursor — still convention-legal.
|
||||
## Adding pagination to a new endpoint
|
||||
|
||||
1. Use `encodeCursor`/`decodeCursor`/`normalizeLimit`/`normalizeOffset` from
|
||||
`src/backend/util/pagination.ts` — `sealCursor`/`openCursor` when the
|
||||
cursor wraps a global id.
|
||||
`src/backend/util/pagination.ts`.
|
||||
2. Push equality filters into the query so pages and counts operate on the
|
||||
true result set; only genuinely per-actor filtering may remain post-query
|
||||
(short-pages rule covers it).
|
||||
|
||||
+26
-124
@@ -35,12 +35,6 @@ Optional services (compose profile `ai`, opt-in):
|
||||
| `puter-ollama` | `ollama/ollama` | Local LLM provider (CPU; GPU passthrough opt-in) |
|
||||
| `puter-ollama-init` | `ollama/ollama` | One-shot — pulls the default model (`tinyllama`) on first boot |
|
||||
|
||||
Optional services (compose profile `smtp`, opt-in):
|
||||
|
||||
| Container | Image | Role |
|
||||
| ------------ | ------------------------ | -------------------------------------------------------- |
|
||||
| `puter-smtp` | `ghcr.io/heyputer/puter` | Receives mail over SMTP and files it into user mailboxes |
|
||||
|
||||
State lives under `./puter/data/<service>/`.
|
||||
|
||||
---
|
||||
@@ -150,16 +144,6 @@ Why these knobs:
|
||||
- `providers.ollama.enabled: false` — Puter auto-probes a local Ollama at `127.0.0.1:11434` by default; without one running you'd see `ECONNREFUSED` on every boot. To run a bundled Ollama, see [Optional: local LLM (Ollama)](#optional-local-llm-ollama) below.
|
||||
- `s3.s3Config.forcePathStyle: true` — RustFS / MinIO / fauxqs need path-style URLs (`<endpoint>/<bucket>`). Real AWS S3 wants virtual-hosted (`<bucket>.<endpoint>`) — drop this flag (or set `false`) when you swap to real S3.
|
||||
- `s3.s3Config.publicEndpoint` — `endpoint` (`http://s3:9000`) only resolves inside the docker network; presigned upload/download URLs handed to the browser need a host-reachable URL. Caddy routes the `s3.<domain>` subdomain to RustFS internally and preserves the Host header end-to-end (required for S3 signature validation), so the browser hits the same port/protocol as the rest of the app — no separate published port, no mixed-content surprises when you turn on TLS. Switch to `https://s3.<your-domain>` once you enable TLS in Step 3. Real AWS S3 doesn't need this — its endpoint is already public; drop the field entirely.
|
||||
- **Bucket CORS** — the browser uploads file bytes straight to RustFS with a cross-origin `PUT` to the presigned URL (Dev Center deploy, `puter.fs.upload`). That only works if the bucket answers the `OPTIONS` preflight with `Access-Control-Allow-*` headers. The `s3-init` container applies those rules on every boot (`put-bucket-cors`, origins `*` — the signed URL is the credential, and the SDK sends no cookies). If deploys fail with `No 'Access-Control-Allow-Origin' header is present`, re-run it: `docker compose run --rm s3-init`. Verify with:
|
||||
|
||||
```bash
|
||||
curl -sk -X OPTIONS "https://s3.<domain>/<bucket>/test" \
|
||||
-H "Origin: https://<domain>" \
|
||||
-H "Access-Control-Request-Method: PUT" \
|
||||
-H "Access-Control-Request-Headers: content-type" -D - -o /dev/null | grep -i access-control
|
||||
```
|
||||
|
||||
You should see `access-control-allow-origin`, `allow-methods: GET, HEAD, PUT, POST, DELETE`, `allow-headers: *`, and `max-age: 3600`.
|
||||
- `trust_proxy: 1` — Caddy terminates TLS and forwards `X-Forwarded-For`. Without this, `req.ip` is the docker-network address of the Caddy container instead of the real client IP, which breaks rate limiting and IP-based audit logs. `1` = one trusted hop (Caddy). Bump to `2` if you put Cloudflare in front of Caddy; never set `true` (it trusts every hop and makes XFF forgeable).
|
||||
|
||||
> If you ever change `MARIADB_PASSWORD` after first boot, `.env` alone won't update MariaDB — its credentials are baked into `./puter/data/mariadb/` on first init. Either rotate the password inside MariaDB by hand or `docker compose down && rm -rf ./puter/data/mariadb` to start fresh.
|
||||
@@ -216,7 +200,7 @@ Drop the resulting `fullchain.pem` and `privkey.pem` into `./puter/tls/`.
|
||||
```
|
||||
…and update the S3 public endpoint:
|
||||
```json
|
||||
"s3": { "s3Config": { "publicEndpoint": "https://s3.puter.localhost", ... } }
|
||||
"s3": { "s3Config": { "publicEndpoint": "https://s3.puter.local", ... } }
|
||||
```
|
||||
|
||||
## Running behind your own reverse proxy
|
||||
@@ -263,15 +247,15 @@ Healthy startup:
|
||||
|
||||
```
|
||||
[config] override from /etc/puter/config.json
|
||||
[mysql] running migrations from /opt/puter/dist/src/backend/clients/database/migrations/mysql: <n> file(s)
|
||||
[mysql] running migrations from /opt/puter/dist/src/backend/clients/database/migrations/mysql: 2 file(s)
|
||||
[mysql] applied mysql_mig_1.sql (...)
|
||||
...
|
||||
[mysql] applied mysql_mig_2.sql (9 statements)
|
||||
```
|
||||
|
||||
Then open **<https://puter.localhost>** (or `http://` if you skipped TLS). Login is `admin`; its temporary password is printed in the puter container logs on every boot until you change it:
|
||||
Then open **<https://puter.local>** (or `http://` if you skipped TLS). Login is `admin` — the temp password is printed once in the puter container logs on first boot:
|
||||
|
||||
```bash
|
||||
docker compose logs puter | grep "password for admin"
|
||||
docker compose logs puter | grep tmp_password
|
||||
```
|
||||
|
||||
Change it in Settings after first login.
|
||||
@@ -394,14 +378,14 @@ Full provider list (chat, image, video, TTS, OCR) is in the template.
|
||||
|
||||
### Per-user storage quota
|
||||
|
||||
Storage is unlimited by default (bounded by host disk). To cap each user:
|
||||
Default is 100 MB per user.
|
||||
|
||||
```json
|
||||
"storage_capacity": 5368709120, // 5 GB
|
||||
"is_storage_limited": true
|
||||
```
|
||||
|
||||
`storage_capacity` defaults to 100 MB.
|
||||
Set `is_storage_limited: false` for unlimited (bounded by host disk).
|
||||
|
||||
### Usage metering and budgets
|
||||
|
||||
@@ -413,8 +397,8 @@ deletion, stay available so an account can always see what it has and clear it.
|
||||
|
||||
On a self-hosted install this is almost certainly not what you want. There is
|
||||
nowhere to buy more, so accounts are held to the free monthly allowance
|
||||
(US$0.50 of measured cost for a registered account, roughly 2.5 GiB of
|
||||
downloads) and start being refused after that. Turn it off:
|
||||
(US$0.25 of measured cost — roughly 2 GiB of downloads) and start being refused
|
||||
after that. Turn it off:
|
||||
|
||||
```json
|
||||
"unlimitedMetering": true
|
||||
@@ -424,6 +408,12 @@ Every account then resolves to an unlimited policy. Usage is still recorded, so
|
||||
the dashboard still shows what is being consumed; nothing is ever refused for
|
||||
lack of budget.
|
||||
|
||||
For API testing, you can also set `"unlimitedMetering": true` in your
|
||||
ignored runtime config: `puter/config/config.json` for Docker, or the repository's
|
||||
`config.json` for `npm start`. Merge it into the existing config and restart Puter.
|
||||
This includes guest accounts and applies to the whole local instance. Keep normal
|
||||
metering settings when testing budget or subscription enforcement.
|
||||
|
||||
To keep the budgets but stop them blocking anything — recording only:
|
||||
|
||||
```json
|
||||
@@ -464,10 +454,9 @@ temporary or permanent one.
|
||||
"disable_user_signup": true
|
||||
```
|
||||
|
||||
### Block disposable email domains
|
||||
### Block disposable email TLDs
|
||||
|
||||
Addresses on these domains (suffix match, so subdomains too) can't sign up or be
|
||||
set as an account's email.
|
||||
Only enforced when `env: "prod"`.
|
||||
|
||||
```json
|
||||
"blockedEmailDomains": ["mailinator.com", "tempmail.com", "guerrillamail.com"]
|
||||
@@ -514,95 +503,6 @@ Without `--profile ai`, the `ollama` containers stay down and Puter (with `enabl
|
||||
|
||||
For GPU acceleration (NVIDIA), uncomment the `deploy:` block under the `ollama` service in [docker-compose.yml](../docker-compose.yml). Requires `nvidia-container-toolkit` on the host.
|
||||
|
||||
## Optional: receiving mail over SMTP
|
||||
|
||||
Puter can give each account a mailbox at a domain you control. Two pieces are
|
||||
involved, and they are configured together under `userEmail`:
|
||||
|
||||
- an **ingress endpoint** (`POST /email/ingress`) that files a message into the
|
||||
recipient's `~/.mail`. It exists whenever `userEmail.secret` is set.
|
||||
- the **`puter-smtp` receiver**, which accepts mail from other mail servers and
|
||||
hands each message to that endpoint. It runs as its own process, behind the
|
||||
`smtp` compose profile.
|
||||
|
||||
> **Read this before you expose it.** An inbound mail server is unauthenticated
|
||||
> by nature: anyone on the internet may hand it a message for any address in
|
||||
> `localDomains`. Puter performs **no** spam filtering and **no** SPF, DKIM or
|
||||
> DMARC checks — whatever is delivered is stored in the recipient's mailbox and
|
||||
> counts against their storage. If that is not acceptable, put a filtering relay
|
||||
> in front of it and point `localDomains` at that instead.
|
||||
|
||||
1. Configure it in `puter/config/config.json`:
|
||||
|
||||
```json
|
||||
"userEmail": {
|
||||
"secret": "a long random string",
|
||||
"localServer": true,
|
||||
"localDomains": ["example.com"]
|
||||
}
|
||||
```
|
||||
|
||||
`secret` is shared between the two halves; generate it with
|
||||
`openssl rand -hex 32`. `localDomains` lists the domains this server accepts
|
||||
mail for — anything else is refused at `RCPT TO`, which is what stops it
|
||||
being used as an open relay. It has no default, and the receiver refuses to
|
||||
start without it.
|
||||
|
||||
The receiver posts to `<api_base_url>/email/ingress` unless you set
|
||||
`localIngressUrl`. The endpoint is served on the `api` subdomain, so if you
|
||||
point it at an internal address to skip the reverse proxy, name that virtual
|
||||
host too or the request will 404:
|
||||
|
||||
```json
|
||||
"localIngressUrl": "http://puter:4100/email/ingress",
|
||||
"localIngressHost": "api.example.com"
|
||||
```
|
||||
|
||||
`localIngressHost` defaults to the host of `api_base_url`, which is usually
|
||||
what you want.
|
||||
|
||||
2. Bring it up with the `smtp` profile:
|
||||
|
||||
```bash
|
||||
docker compose --profile smtp up -d
|
||||
docker compose logs -f puter-smtp
|
||||
```
|
||||
|
||||
3. Point DNS at it:
|
||||
|
||||
```dns
|
||||
mx.example.com. A 203.0.113.10
|
||||
example.com. MX 10 mx.example.com.
|
||||
```
|
||||
|
||||
Also set a `PTR` record for the address that resolves back to
|
||||
`mx.example.com` — many senders refuse mail from a server without one.
|
||||
|
||||
Inbound TCP port 25 must reach the host. **Most cloud providers block port
|
||||
25 by default and residential ISPs block it permanently**; if you cannot get
|
||||
it unblocked, this will not work no matter how it is configured. The
|
||||
container itself binds 2525 (it runs unprivileged) and compose publishes
|
||||
`${SMTP_PORT:-25}` onto it, so you can move the host port if you need to.
|
||||
|
||||
A message is held in memory while it is being delivered, so peak memory is
|
||||
bounded by `localMaxClients` (default 20) times the 25 MiB message ceiling.
|
||||
Lower `localMaxClients` on a small host. There is no queue and no retry
|
||||
schedule — if delivery fails, the sending server is told to try again later and
|
||||
retrying is its job.
|
||||
|
||||
Mail is stored as `message/rfc822` objects under `~/.mail/objects/<date>/` and
|
||||
read back through `puter.email` in the SDK.
|
||||
|
||||
Every setting is documented in
|
||||
[config.template.jsonc](../config.template.jsonc). Two deliberate limits, both
|
||||
in the name of keeping the self-hosted path simple: it does not offer STARTTLS,
|
||||
so mail arrives in the clear, and it applies no per-sender rate limiting. Put a
|
||||
filtering relay in front if either matters to you.
|
||||
|
||||
Note this is a *receiving* server only: it never sends, and it must not be used
|
||||
for authenticated submission. Outbound transactional mail is the separate
|
||||
`email` block above.
|
||||
|
||||
## Building from source instead of pulling
|
||||
|
||||
To run a local build against the full stack, use a source checkout and complete
|
||||
@@ -618,10 +518,12 @@ services:
|
||||
```
|
||||
|
||||
If that file already exists, merge these settings into its `puter` service.
|
||||
Compose merges the override automatically: `build.context: .` builds this
|
||||
checkout, and `pull_policy: never` stops it pulling the published image. Git
|
||||
ignores the override, so local build settings stay out of pull requests and
|
||||
don't conflict with `docker-compose.yml` when you pull updates.
|
||||
Compose loads and merges the override automatically. `build.context: .` selects
|
||||
this checkout, and `pull_policy: never` prevents pulling the published Puter image.
|
||||
|
||||
Use this override for local build settings instead of editing `docker-compose.yml`.
|
||||
The override is ignored by Git, keeping local configuration out of pull requests
|
||||
and avoiding conflicts in the shared Compose file when pulling updates.
|
||||
|
||||
From the repository root, build and start the stack:
|
||||
|
||||
@@ -660,11 +562,11 @@ Migrations re-apply idempotently across pulls. Volumes are preserved.
|
||||
**Site loads but I get a 502 / "Bad Gateway" from Caddy.**
|
||||
The puter container failed to come up. `docker compose logs puter` will tell you which dependency rejected it (most often DB password mismatch between `.env` and `config.json`).
|
||||
|
||||
**Can't find the admin password.**
|
||||
It's logged on every boot until you change it: `docker compose logs puter | grep "password for admin"`. After login, change it in Settings.
|
||||
**Login screen says "admin password not set".**
|
||||
First-boot temp password is logged once. Find it: `docker compose logs puter | grep "tmp_password"`. After login, change it in Settings.
|
||||
|
||||
**Healthcheck reports unhealthy but the site works.**
|
||||
The compose healthcheck requests `http://localhost:4100/` from inside the container. If you changed `port` or set `allow_all_host_values: false`, the check fails while the site itself is fine.
|
||||
The healthcheck hits `puter.localhost:4100/test` from inside the container. If you changed `domain` or `port`, the check still uses defaults. The site itself is fine.
|
||||
|
||||
**Nothing resolves at `puter.example.com` after DNS changes.**
|
||||
DNS propagates slowly. `dig puter.example.com` and `dig api.puter.example.com` should both return your server IP. If not, give it 5–60 minutes.
|
||||
|
||||
+2
-50
@@ -139,10 +139,8 @@ services:
|
||||
start_period: 5s
|
||||
|
||||
s3-init:
|
||||
# One-shot container that creates the `puter-local` bucket and
|
||||
# applies its CORS rules on every boot (put-bucket-cors is
|
||||
# idempotent). Without CORS, browser uploads to presigned URLs
|
||||
# fail the preflight check.
|
||||
# One-shot container that creates the `puter-local` bucket on first
|
||||
# boot. Exits 0 once the bucket exists; stays exited 0 thereafter.
|
||||
image: amazon/aws-cli:latest
|
||||
container_name: puter-s3-init
|
||||
depends_on:
|
||||
@@ -165,26 +163,6 @@ services:
|
||||
echo "creating bucket $$bucket"
|
||||
aws --endpoint-url "$$endpoint" s3 mb "s3://$$bucket"
|
||||
fi
|
||||
# Browser uploads (Dev Center deploy, puter.fs.upload) PUT to
|
||||
# presigned URLs cross-origin, so the bucket must answer the
|
||||
# OPTIONS preflight. Origins are open because the presigned URL
|
||||
# itself is the credential and the SDK sends no cookies.
|
||||
cat > /tmp/cors.json <<'EOF'
|
||||
{
|
||||
"CORSRules": [
|
||||
{
|
||||
"AllowedOrigins": ["*"],
|
||||
"AllowedMethods": ["GET", "HEAD", "PUT", "POST", "DELETE"],
|
||||
"AllowedHeaders": ["*"],
|
||||
"ExposeHeaders": ["ETag", "x-amz-request-id"],
|
||||
"MaxAgeSeconds": 3600
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
aws --endpoint-url "$$endpoint" s3api put-bucket-cors \
|
||||
--bucket "$$bucket" --cors-configuration file:///tmp/cors.json
|
||||
echo "bucket CORS configured"
|
||||
restart: "no"
|
||||
|
||||
# ── Optional: local LLM ───────────────────────────────────────────
|
||||
@@ -286,32 +264,6 @@ services:
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
# Inbound mail. Behind the `smtp` profile, so it only starts when asked for:
|
||||
# docker compose --profile smtp up -d
|
||||
# Accepts SMTP from other mail servers and hands each message to the ingress
|
||||
# endpoint, which files it into the recipient's mailbox. Needs `userEmail`
|
||||
# configured — see doc/self-hosting.md#optional-receiving-mail-over-smtp.
|
||||
puter-smtp:
|
||||
profiles: ["smtp"]
|
||||
image: ghcr.io/heyputer/puter:main
|
||||
pull_policy: always
|
||||
container_name: puter-smtp
|
||||
restart: unless-stopped
|
||||
command: ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/smtp/index.js"]
|
||||
depends_on:
|
||||
puter:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
# The container runs unprivileged and binds 2525; 25 is what an MX record
|
||||
# means, and many providers block it.
|
||||
- "${SMTP_PORT:-25}:2525"
|
||||
environment:
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
volumes:
|
||||
# The same config.json the backend reads; only `userEmail` applies here.
|
||||
- ./puter/config:/etc/puter:z
|
||||
|
||||
caddy:
|
||||
image: caddy:2.11-alpine
|
||||
container_name: puter-caddy
|
||||
|
||||
+51
-23
@@ -1,4 +1,5 @@
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import type { Actor } from '@heyputer/backend/src/core/actor';
|
||||
import { HttpError } from '@heyputer/backend/src/core/http';
|
||||
import { PuterDriver } from '@heyputer/backend/src/drivers/types';
|
||||
import type {
|
||||
@@ -132,30 +133,51 @@ export class AppTelemetryDriver extends PuterDriver {
|
||||
const owner = await this.stores.user.getById(ownerId);
|
||||
if (!owner?.uuid) throw new HttpError(404, 'App owner not found');
|
||||
|
||||
// `/drivers/call` requires auth, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
let ownsApp = false;
|
||||
try {
|
||||
ownsApp = await this.services.permission.check(
|
||||
actor,
|
||||
`apps-of-user:${owner.uuid}:write`,
|
||||
);
|
||||
} catch {
|
||||
// A failed check is a denial.
|
||||
}
|
||||
const actor = Context.get('actor');
|
||||
if (!actor) throw new HttpError(401, 'Authentication required');
|
||||
const ownsApp = await this.services.permission
|
||||
.check(actor as Actor, `apps-of-user:${owner.uuid}:write`)
|
||||
.catch(() => false);
|
||||
if (!ownsApp) throw new HttpError(403, 'Permission denied');
|
||||
|
||||
// An email surfaces only for a user who granted *this app*
|
||||
// `user:<their-uuid>:email:read` — the grant `puter.perms.requestEmail()`
|
||||
// obtains and `whoami` honours. Authenticating alone doesn't share it.
|
||||
const users = await this.stores.permission.listAppAuthenticatedUsers(
|
||||
(app as { id: number }).id,
|
||||
{ limit: safeLimit, offset: safeOffset },
|
||||
);
|
||||
return users.map((u) =>
|
||||
u.emailShared
|
||||
? { user: u.username, user_uuid: u.uuid, user_email: u.email }
|
||||
: { user: u.username, user_uuid: u.uuid },
|
||||
const appId = (app as { id: number }).id;
|
||||
|
||||
const users = (await this.clients.db.read(
|
||||
`SELECT u.id, u.username, u.uuid, u.email FROM user_to_app_permissions p
|
||||
INNER JOIN ${this.clients.db.quoteIdentifier('user')} u ON p.user_id = u.id
|
||||
WHERE p.permission = 'flag:app-is-authenticated' AND p.app_id = ?
|
||||
ORDER BY (p.dt IS NOT NULL), p.dt, p.user_id
|
||||
LIMIT ? OFFSET ?`,
|
||||
[appId, safeLimit, safeOffset],
|
||||
)) as Array<{
|
||||
id: number;
|
||||
username: string;
|
||||
uuid: string;
|
||||
email: string | null;
|
||||
}>;
|
||||
|
||||
// Only surface a user's email if *that user* granted this app the
|
||||
// `user:<their-uuid>:email:read` permission — the same grant
|
||||
// `puter.perms.requestEmail()` obtains and `whoami` honours. This is a
|
||||
// per-user check keyed on the app (not the calling owner-actor): a
|
||||
// user may have authenticated into the app without sharing their
|
||||
// email. Resolve the whole page in one query.
|
||||
const emailPermitted = new Set<number>();
|
||||
if (users.length > 0) {
|
||||
const permStrings = users.map((u) => `user:${u.uuid}:email:read`);
|
||||
const placeholders = permStrings.map(() => '?').join(', ');
|
||||
const grants = (await this.clients.db.read(
|
||||
`SELECT user_id FROM user_to_app_permissions
|
||||
WHERE app_id = ? AND permission IN (${placeholders})`,
|
||||
[appId, ...permStrings],
|
||||
)) as Array<{ user_id: number }>;
|
||||
for (const g of grants) emailPermitted.add(g.user_id);
|
||||
}
|
||||
|
||||
return users.map((e) =>
|
||||
emailPermitted.has(e.id)
|
||||
? { user: e.username, user_uuid: e.uuid, user_email: e.email }
|
||||
: { user: e.username, user_uuid: e.uuid },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -168,7 +190,13 @@ export class AppTelemetryDriver extends PuterDriver {
|
||||
const app = await this.stores.app.getByUid(app_uuid);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
|
||||
return this.stores.permission.countAppUsers((app as { id: number }).id);
|
||||
const [row] = (await this.clients.db.read(
|
||||
`SELECT COUNT(*) AS n FROM user_to_app_permissions
|
||||
WHERE permission = 'flag:app-is-authenticated' AND app_id = ?`,
|
||||
[(app as { id: number }).id],
|
||||
)) as Array<{ n: number }>;
|
||||
|
||||
return row?.n ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -74,6 +74,15 @@ const grantInstalled = async (appId: number, userId: number) => {
|
||||
};
|
||||
|
||||
describe('installedApps extension — handleInstalledApps', () => {
|
||||
it('throws HttpError(401) when no actor is on the context', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
runWithContext({ actor: undefined }, () =>
|
||||
handleInstalledApps(makeReq({}), res),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it('throws HttpError(400) when orderBy is not in the allowlist', async () => {
|
||||
const user = await seedUser();
|
||||
const { res } = makeRes();
|
||||
|
||||
+43
-16
@@ -2,41 +2,68 @@ import type { Request, Response } from 'express';
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { HttpError } from '@heyputer/backend/src/core/http';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import {
|
||||
INSTALLED_APPS_ORDERS,
|
||||
type InstalledAppsOrder,
|
||||
} from '@heyputer/backend/src/stores/permission/PermissionStore.js';
|
||||
import {
|
||||
getAppIconCdnUrl,
|
||||
getAppIconUrl,
|
||||
} from '@heyputer/backend/src/util/appIcon.js';
|
||||
|
||||
const stores = extension.import('store');
|
||||
const clients = extension.import('client');
|
||||
|
||||
const ALLOWED_ORDER_BY = [
|
||||
'id',
|
||||
'name',
|
||||
'uid',
|
||||
'title',
|
||||
'installed_at',
|
||||
] as const;
|
||||
const ORDER_BY_FIELD_MAP: Record<string, string> = {
|
||||
id: 'apps.id',
|
||||
name: 'apps.name',
|
||||
uid: 'apps.uid',
|
||||
title: 'apps.title',
|
||||
installed_at: 'installed_at',
|
||||
};
|
||||
|
||||
export const handleInstalledApps = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
// Behind `requireUserActor`, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
const actor = Context.get('actor');
|
||||
if (!actor?.user?.id) throw new HttpError(401, 'Authentication required');
|
||||
|
||||
const orderBy = String(
|
||||
req.query.orderBy ?? 'installed_at',
|
||||
) as InstalledAppsOrder;
|
||||
if (!INSTALLED_APPS_ORDERS.includes(orderBy)) {
|
||||
const orderBy = String(req.query.orderBy ?? 'installed_at');
|
||||
if (!(ALLOWED_ORDER_BY as readonly string[]).includes(orderBy)) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`Invalid orderBy. Allowed: ${INSTALLED_APPS_ORDERS.join(', ')}`,
|
||||
`Invalid orderBy. Allowed: ${ALLOWED_ORDER_BY.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
const page = Math.max(Number(req.query.page) || 1, 1);
|
||||
const limit = Math.min(Math.max(Number(req.query.limit) || 100, 1), 100);
|
||||
const offset = (page - 1) * limit;
|
||||
const installedApps = await stores.permission.listAppsGrantedByUser(
|
||||
actor.user.id!,
|
||||
{ orderBy, descending: Boolean(req.query.desc), limit, offset },
|
||||
);
|
||||
const orderByField = ORDER_BY_FIELD_MAP[orderBy];
|
||||
const sortDirection = req.query.desc ? 'DESC' : 'ASC';
|
||||
|
||||
const installedApps = (await clients.db.read(
|
||||
`SELECT
|
||||
apps.name,
|
||||
apps.uid,
|
||||
apps.title,
|
||||
apps.description,
|
||||
apps.icon,
|
||||
apps.index_url,
|
||||
apps.owner_user_id,
|
||||
MIN(perm.dt) AS installed_at
|
||||
FROM apps
|
||||
LEFT JOIN user_to_app_permissions AS perm ON apps.id = perm.app_id
|
||||
WHERE perm.user_id = ?
|
||||
GROUP BY apps.id, apps.name, apps.uid, apps.title, apps.description
|
||||
ORDER BY ${orderByField} ${sortDirection}
|
||||
LIMIT ?
|
||||
OFFSET ?`,
|
||||
[actor.user.id, limit, offset],
|
||||
)) as Array<Record<string, unknown>>;
|
||||
|
||||
const apiBaseUrl = extension.config.api_base_url as string | undefined;
|
||||
res.json(
|
||||
|
||||
+21
-202
@@ -8,7 +8,6 @@ import {
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { makeActor, type Actor } from '../src/backend/core/actor.ts';
|
||||
import { runWithContext } from '../src/backend/core/context.ts';
|
||||
import { PuterServer } from '../src/backend/server.ts';
|
||||
import { setupTestServer } from '../src/backend/testUtil.ts';
|
||||
@@ -63,6 +62,15 @@ const seedUser = async () => {
|
||||
};
|
||||
|
||||
describe('metering extension — handleMeteringUsage', () => {
|
||||
it('throws HttpError(401) when no user actor is on the context', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
runWithContext({ actor: undefined }, () =>
|
||||
handleMeteringUsage(makeReq(), res),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it('returns usage details merged with allowanceInfo for an authenticated user', async () => {
|
||||
const user = await seedUser();
|
||||
const { res, captured } = makeRes();
|
||||
@@ -80,83 +88,21 @@ describe('metering extension — handleMeteringUsage', () => {
|
||||
(captured.body as Record<string, unknown>).allowanceInfo,
|
||||
).toBeDefined();
|
||||
});
|
||||
|
||||
describe('usage split across apps', () => {
|
||||
const seedUsage = async () => {
|
||||
const user = await seedUser();
|
||||
const owner = { uuid: user.uuid, id: user.id as number };
|
||||
const metering = server.services.metering;
|
||||
await metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: 'app-mine' } }),
|
||||
'kv:read',
|
||||
1,
|
||||
10,
|
||||
);
|
||||
await metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: 'app-other' } }),
|
||||
'kv:write',
|
||||
1,
|
||||
20,
|
||||
);
|
||||
await metering.incrementUsage(
|
||||
makeActor({ user: owner }),
|
||||
'kv:list',
|
||||
1,
|
||||
40,
|
||||
);
|
||||
await server.stores.meteringBuffer.flushCycle();
|
||||
return owner;
|
||||
};
|
||||
|
||||
const usageAs = async (actor: Actor) => {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () =>
|
||||
handleMeteringUsage(makeReq(), res),
|
||||
);
|
||||
return captured.body as {
|
||||
usage: Record<string, unknown> & { total: number };
|
||||
appTotals: Record<string, unknown>;
|
||||
allowanceInfo: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
|
||||
it('reports only the calling app’s own usage to an app actor', async () => {
|
||||
const owner = await seedUsage();
|
||||
const body = await usageAs(
|
||||
makeActor({ user: owner, app: { uid: 'app-mine' } }),
|
||||
);
|
||||
|
||||
expect(body.usage.total).toBe(10);
|
||||
expect(Object.keys(body.usage).sort()).toEqual([
|
||||
'kv:read',
|
||||
'total',
|
||||
]);
|
||||
expect(body.appTotals).toEqual({
|
||||
'app-mine': { total: 10, count: 1 },
|
||||
});
|
||||
expect(body.allowanceInfo).toMatchObject({
|
||||
remaining: expect.any(Number),
|
||||
monthUsageAllowance: expect.any(Number),
|
||||
});
|
||||
});
|
||||
|
||||
it('still reports the whole account to the user', async () => {
|
||||
const owner = await seedUsage();
|
||||
const body = await usageAs(makeActor({ user: owner }));
|
||||
|
||||
expect(body.usage.total).toBe(70);
|
||||
expect(body.usage).toHaveProperty('kv:read');
|
||||
expect(body.usage).toHaveProperty('kv:write');
|
||||
expect(body.usage).toHaveProperty('kv:list');
|
||||
expect(body.appTotals).toMatchObject({
|
||||
'app-mine': { total: 10, count: 1 },
|
||||
'app-other': { total: 20, count: 1 },
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('metering extension — handleMeteringUsageForApp', () => {
|
||||
it('throws HttpError(401) when no user actor is on the context', async () => {
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
runWithContext({ actor: undefined }, () =>
|
||||
handleMeteringUsageForApp(
|
||||
makeReq({ params: { appIdOrName: 'any' } }),
|
||||
res,
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 401 });
|
||||
});
|
||||
|
||||
it('throws HttpError(400) when no appId is supplied', async () => {
|
||||
const user = await seedUser();
|
||||
const { res } = makeRes();
|
||||
@@ -186,96 +132,6 @@ describe('metering extension — handleMeteringUsageForApp', () => {
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
describe('scoped to the calling app', () => {
|
||||
const appUsageAs = async (actor: Actor, appIdOrName: string) => {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () =>
|
||||
handleMeteringUsageForApp(
|
||||
makeReq({ params: { appIdOrName } }),
|
||||
res,
|
||||
),
|
||||
);
|
||||
return captured.body as { total: number };
|
||||
};
|
||||
|
||||
const seedApps = async () => {
|
||||
const user = await seedUser();
|
||||
const owner = { uuid: user.uuid, id: user.id as number };
|
||||
const mine = `app-${uuidv4()}`;
|
||||
const other = `app-${uuidv4()}`;
|
||||
await server.services.metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: mine } }),
|
||||
'kv:read',
|
||||
1,
|
||||
10,
|
||||
);
|
||||
await server.services.metering.incrementUsage(
|
||||
makeActor({ user: owner, app: { uid: other } }),
|
||||
'kv:write',
|
||||
1,
|
||||
20,
|
||||
);
|
||||
await server.stores.meteringBuffer.flushCycle();
|
||||
return {
|
||||
owner,
|
||||
mine,
|
||||
other,
|
||||
asMine: makeActor({ user: owner, app: { uid: mine } }),
|
||||
};
|
||||
};
|
||||
|
||||
it('lets an app read its own usage', async () => {
|
||||
const { mine, asMine } = await seedApps();
|
||||
expect((await appUsageAs(asMine, mine)).total).toBe(10);
|
||||
});
|
||||
|
||||
it('refuses an app another app’s usage', async () => {
|
||||
const { other, asMine } = await seedApps();
|
||||
await expect(appUsageAs(asMine, other)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses an app another app’s usage by name', async () => {
|
||||
const { owner, other, asMine } = await seedApps();
|
||||
const name = `other-${uuidv4()}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[other, name, name, `https://${name}.example/`, owner.id],
|
||||
);
|
||||
await expect(appUsageAs(asMine, name)).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses an app the usage outside any app', async () => {
|
||||
const { asMine } = await seedApps();
|
||||
await expect(appUsageAs(asMine, 'os-global')).rejects.toMatchObject(
|
||||
{ statusCode: 403 },
|
||||
);
|
||||
});
|
||||
|
||||
it('still lets the user read any of their apps', async () => {
|
||||
const { owner, mine, other } = await seedApps();
|
||||
const asUser = makeActor({ user: owner });
|
||||
expect((await appUsageAs(asUser, mine)).total).toBe(10);
|
||||
expect((await appUsageAs(asUser, other)).total).toBe(20);
|
||||
});
|
||||
|
||||
it('lets the user read their usage outside any app', async () => {
|
||||
const { owner } = await seedApps();
|
||||
const asUser = makeActor({ user: owner });
|
||||
await server.services.metering.incrementUsage(
|
||||
asUser,
|
||||
'kv:read',
|
||||
1,
|
||||
5,
|
||||
);
|
||||
await server.stores.meteringBuffer.flushCycle();
|
||||
expect((await appUsageAs(asUser, 'os-global')).total).toBe(5);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('metering extension — handleMeteringGlobalUsage', () => {
|
||||
@@ -309,41 +165,4 @@ describe('metering extension — handleMeteringAllCosts', () => {
|
||||
// per request).
|
||||
expect(costsA).toBe(costsB);
|
||||
});
|
||||
|
||||
it('refreshes a warm catalogue at the UTC price boundary', async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
try {
|
||||
vi.setSystemTime(new Date('2026-11-30T23:59:59Z'));
|
||||
const before = makeRes();
|
||||
await handleMeteringAllCosts(makeReq(), before.res);
|
||||
vi.setSystemTime(new Date('2026-12-01T00:00:00Z'));
|
||||
const after = makeRes();
|
||||
await handleMeteringAllCosts(makeReq(), after.res);
|
||||
expect(
|
||||
(after.captured.body as { costs: unknown[] }).costs,
|
||||
).not.toBe((before.captured.body as { costs: unknown[] }).costs);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
// A scheduled price change has to reach the catalogue without a restart.
|
||||
it('collects the catalogue again once the cache expires', async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
try {
|
||||
vi.setSystemTime(new Date('2026-12-02T00:00:00Z'));
|
||||
const a = makeRes();
|
||||
const b = makeRes();
|
||||
await handleMeteringAllCosts(makeReq(), a.res);
|
||||
vi.setSystemTime(Date.now() + 10 * 60 * 1000 + 1);
|
||||
await handleMeteringAllCosts(makeReq(), b.res);
|
||||
|
||||
const costsA = (a.captured.body as { costs: unknown[] }).costs;
|
||||
const costsB = (b.captured.body as { costs: unknown[] }).costs;
|
||||
expect(costsB).not.toBe(costsA);
|
||||
expect(costsB).toEqual(costsA);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
+20
-20
@@ -6,7 +6,6 @@ import {
|
||||
servicesContainers,
|
||||
} from '@heyputer/backend/src/exports';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import { GLOBAL_APP_KEY } from '@heyputer/backend/src/services/metering/consts';
|
||||
import {
|
||||
creditMultiplierFrom,
|
||||
toCredits,
|
||||
@@ -14,13 +13,11 @@ import {
|
||||
import type { Request, Response } from 'express';
|
||||
|
||||
const services = extension.import('service');
|
||||
const stores = extension.import('store');
|
||||
const clients = extension.import('client');
|
||||
|
||||
// Catalogues change on deploy or on a scheduled price change, so the cache
|
||||
// expires rather than living as long as the process.
|
||||
const ALL_COSTS_TTL_MS = 10 * 60 * 1000;
|
||||
// Cached on first request — the underlying cost catalogues are baked into
|
||||
// driver/controller source so they only change on deploy.
|
||||
let cachedAllCosts: Record<string, unknown>[] | null = null;
|
||||
let allCostsExpireAt = 0;
|
||||
|
||||
function collectAllCosts(): Record<string, unknown>[] {
|
||||
const all: Record<string, unknown>[] = [];
|
||||
@@ -97,8 +94,8 @@ export const handleMeteringUsage = async (
|
||||
_req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
// Behind `requireAuth`, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
const actor = Context.get('actor');
|
||||
if (!actor?.user) throw new HttpError(401, 'Authentication required');
|
||||
|
||||
const [actorUsage, allowanceInfo] = await Promise.all([
|
||||
services.metering.getActorCurrentMonthUsageDetails(actor),
|
||||
@@ -148,18 +145,23 @@ export const handleMeteringUsageForApp = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
// Behind `requireAuth`, so there is always an actor.
|
||||
const actor = Context.get('actor')!;
|
||||
const actor = Context.get('actor');
|
||||
if (!actor?.user) throw new HttpError(401, 'Authentication required');
|
||||
|
||||
let appId = String(req.params.appIdOrName ?? '');
|
||||
if (!appId) throw new HttpError(400, 'appId parameter is required');
|
||||
|
||||
// If not a UUID-shaped app UID or the global sentinel, look up by name.
|
||||
// Which apps an actor may read is MeteringService's call, not this route's.
|
||||
if (!appId.startsWith('app-') && appId !== GLOBAL_APP_KEY) {
|
||||
const app = await stores.app.getByName(appId);
|
||||
if (!app) throw new HttpError(404, 'App not found');
|
||||
appId = app.uid;
|
||||
// If not a UUID-shaped app UID, look up by name
|
||||
if (!appId.startsWith('app-')) {
|
||||
const appRows = (await clients.db.read(
|
||||
'SELECT `uid` FROM `apps` WHERE `name` = ? LIMIT 1',
|
||||
[appId],
|
||||
)) as Array<{ uid: string }>;
|
||||
if (appRows.length > 0) {
|
||||
appId = appRows[0].uid;
|
||||
} else {
|
||||
throw new HttpError(404, 'App not found');
|
||||
}
|
||||
}
|
||||
|
||||
const appUsage =
|
||||
@@ -189,14 +191,12 @@ export const handleMeteringGlobalUsage = async (
|
||||
res.json(globalUsage);
|
||||
};
|
||||
|
||||
// A hit walks the registries when the in-memory cache is missing or expired.
|
||||
// First hit walks the registries; subsequent hits serve the in-memory cache.
|
||||
export const handleMeteringAllCosts = async (
|
||||
_req: Request,
|
||||
res: Response,
|
||||
): Promise<void> => {
|
||||
if (!cachedAllCosts || Date.now() >= allCostsExpireAt) {
|
||||
allCostsExpireAt =
|
||||
(Math.floor(Date.now() / ALL_COSTS_TTL_MS) + 1) * ALL_COSTS_TTL_MS;
|
||||
if (!cachedAllCosts) {
|
||||
const raw = collectAllCosts();
|
||||
const multiplier = creditMultiplierFrom(extension.config);
|
||||
// Price the catalogue in the same unit the usage endpoints report:
|
||||
|
||||
+193
-801
File diff suppressed because it is too large.
Load diff
+86
-206
@@ -1,21 +1,15 @@
|
||||
import {
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import type { FSEntryStore } from '@heyputer/backend/src/stores/fs/FSEntryStore';
|
||||
import { isMissingObjectError } from '@heyputer/backend/src/stores/fs/S3ObjectStore';
|
||||
import crypto from 'node:crypto';
|
||||
import type { Readable } from 'node:stream';
|
||||
import sharp from 'sharp';
|
||||
const clients = extension.import('client');
|
||||
const stores = extension.import('store');
|
||||
|
||||
type ThumbnailEntryStore = Pick<FSEntryStore, 'updateEntryThumbnailByUuid'>;
|
||||
|
||||
const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_THUMBNAIL_PIXELS = 64e6;
|
||||
@@ -28,13 +22,8 @@ const THUMBNAIL_KEY_PREFIX = 'thumbnails/';
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
const isUuid = (value: unknown): value is string =>
|
||||
typeof value === 'string' && UUID_PATTERN.test(value);
|
||||
|
||||
// `thumbnails/<entry uuid>/<random uuid>`: the entry segment binds the object
|
||||
// to the entry it was minted for, which is what writes and deletes check.
|
||||
const mintThumbnailKey = (entryUuid: string): string =>
|
||||
`${THUMBNAIL_KEY_PREFIX}${entryUuid}/${crypto.randomUUID()}`;
|
||||
const mintThumbnailKey = (): string =>
|
||||
`${THUMBNAIL_KEY_PREFIX}${crypto.randomUUID()}`;
|
||||
|
||||
/**
|
||||
* Extract the object key from a stored thumbnail pointer, or null when the
|
||||
@@ -42,13 +31,12 @@ const mintThumbnailKey = (entryUuid: string): string =>
|
||||
*
|
||||
* `fsentries.thumbnail` is writable through the FS API, so neither half of the
|
||||
* stored string is trusted: the bucket is discarded (callers always pass their
|
||||
* own) and the key must sit under {@link THUMBNAIL_KEY_PREFIX} as either
|
||||
* `<uuid>` (minted before keys were bound to entries) or `<entry uuid>/<uuid>`.
|
||||
* Honouring an arbitrary key would lend this extension's storage credentials to
|
||||
* whatever object the caller named — in the shared-bucket layout that is every
|
||||
* user's file, since an fs object's key is its fsentry uuid. Bare-uuid keys
|
||||
* from before the prefix fail the check and are treated as absent; they are
|
||||
* indistinguishable from a planted pointer, so there is nothing safer to do
|
||||
* own) and the key must sit under {@link THUMBNAIL_KEY_PREFIX} with a random
|
||||
* uuid. Honouring an arbitrary key would lend this extension's storage
|
||||
* credentials to whatever object the caller named — in the shared-bucket layout
|
||||
* that is every user's file, since an fs object's key is its fsentry uuid.
|
||||
* Legacy bare-uuid thumbnails fail the check and are treated as absent; they
|
||||
* are indistinguishable from a planted pointer, so there is nothing safer to do
|
||||
* with them than stop signing them.
|
||||
*/
|
||||
const resolveThumbnailKey = (pointer: string): string | null => {
|
||||
@@ -70,26 +58,10 @@ const resolveThumbnailKey = (pointer: string): string | null => {
|
||||
key = segments.join('/');
|
||||
}
|
||||
if (!key.startsWith(THUMBNAIL_KEY_PREFIX)) return null;
|
||||
const segments = key.slice(THUMBNAIL_KEY_PREFIX.length).split('/');
|
||||
return segments.length <= 2 && segments.every(isUuid) ? key : null;
|
||||
const id = key.slice(THUMBNAIL_KEY_PREFIX.length);
|
||||
return UUID_PATTERN.test(id) ? key : null;
|
||||
};
|
||||
|
||||
// The entry a resolved key was minted for. Null for an unbound key, whose
|
||||
// holder can't be told apart from a row that copied or planted the pointer.
|
||||
const thumbnailKeyOwner = (key: string): string | null => {
|
||||
const segments = key.slice(THUMBNAIL_KEY_PREFIX.length).split('/');
|
||||
return segments.length === 2 ? segments[0] : null;
|
||||
};
|
||||
|
||||
// Whether a stored thumbnail names an object in our storage. Anything else is
|
||||
// an external image URL the client supplied: shown as given, never copied or
|
||||
// deleted.
|
||||
const isStoragePointer = (pointer: string, bucketEndpoint: string): boolean =>
|
||||
pointer.startsWith('s3://') ||
|
||||
// Legacy format — remove after full migration
|
||||
(pointer.startsWith('https') &&
|
||||
pointer.includes(new URL(bucketEndpoint).hostname));
|
||||
|
||||
// S3 client + bucket config — lazily resolved after boot from config.
|
||||
let s3Client: S3Client | null = null;
|
||||
let s3PresignClient: S3Client | null = null;
|
||||
@@ -139,6 +111,13 @@ function getPresignClient(): S3Client {
|
||||
return resolveClients().presign;
|
||||
}
|
||||
|
||||
function base64ParseDataUrl(dataURL: string) {
|
||||
dataURL = dataURL.slice(5);
|
||||
const mimeType = dataURL.split(';')[0];
|
||||
const data = Buffer.from(dataURL.split(',')[1], 'base64');
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
// Strictly decode a data: URL and validate the decoded image. Encoded-string
|
||||
// length lies about decoded byte count (whitespace, padding) and says nothing
|
||||
// about pixel count — a 2MB PNG can decompress to hundreds of MB of raster.
|
||||
@@ -165,73 +144,16 @@ async function decodeAndValidateThumbnail(
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
// A storage pointer a client hands back after a presigned upload is kept only
|
||||
// if it names a key minted for this entry and an object within the size bound.
|
||||
async function vetUploadedThumbnail(
|
||||
pointer: string,
|
||||
entryUuid: unknown,
|
||||
deps: { s3: S3Client; bucketName: string },
|
||||
): Promise<string | null> {
|
||||
const key = resolveThumbnailKey(pointer);
|
||||
if (!key || thumbnailKeyOwner(key) !== entryUuid) return null;
|
||||
|
||||
let size: number | undefined;
|
||||
try {
|
||||
const head = await deps.s3.send(
|
||||
new HeadObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
);
|
||||
size = head.ContentLength;
|
||||
} catch (err) {
|
||||
if (!isMissingObjectError(err)) {
|
||||
console.warn(
|
||||
'[thumbnails] failed to check uploaded thumbnail',
|
||||
err,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (typeof size !== 'number') return null;
|
||||
if (size > MAX_THUMBNAIL_BYTES) {
|
||||
// Only reachable on a store that doesn't enforce the signed length.
|
||||
// The key is this entry's own, so removing it is safe.
|
||||
try {
|
||||
await deps.s3.send(
|
||||
new DeleteObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[thumbnails] failed to remove oversized thumbnail',
|
||||
err,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return `s3://${deps.bucketName}/${key}`;
|
||||
}
|
||||
|
||||
// -- thumbnail.created -----------------------------------------------
|
||||
// Intercept data-URL thumbnails before they hit the DB: upload to S3
|
||||
// and replace the URL with an s3:// pointer bound to the entry.
|
||||
// and replace the URL with an s3:// pointer.
|
||||
|
||||
export async function handleThumbnailCreated(
|
||||
event: Record<string, unknown>,
|
||||
deps: { s3: S3Client; bucketName: string; bucketEndpoint: string },
|
||||
deps: { s3: S3Client; bucketName: string },
|
||||
): Promise<void> {
|
||||
const url = event.url;
|
||||
if (typeof url !== 'string') return;
|
||||
|
||||
if (!url.startsWith('data:')) {
|
||||
if (isStoragePointer(url, deps.bucketEndpoint)) {
|
||||
event.url = await vetUploadedThumbnail(url, event.uuid, deps);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const entryUuid = event.uuid;
|
||||
if (!isUuid(entryUuid)) {
|
||||
event.url = null;
|
||||
return;
|
||||
}
|
||||
if (typeof url !== 'string' || !url.startsWith('data:')) return;
|
||||
|
||||
const decoded = await decodeAndValidateThumbnail(url);
|
||||
if (!decoded) {
|
||||
@@ -239,7 +161,7 @@ export async function handleThumbnailCreated(
|
||||
return;
|
||||
}
|
||||
|
||||
const key = mintThumbnailKey(entryUuid);
|
||||
const key = mintThumbnailKey();
|
||||
event.url = `s3://${deps.bucketName}/${key}`;
|
||||
|
||||
await deps.s3.send(
|
||||
@@ -268,24 +190,21 @@ export const handleThumbnailUploadPrepare = async (
|
||||
typeof item.contentType === 'string' ? item.contentType.trim() : '';
|
||||
if (!contentType) continue;
|
||||
|
||||
const size = item.size;
|
||||
if (
|
||||
typeof size !== 'number' ||
|
||||
!Number.isInteger(size) ||
|
||||
size < 0 ||
|
||||
size > MAX_THUMBNAIL_BYTES
|
||||
)
|
||||
continue;
|
||||
if (!isUuid(item.item_uid)) continue;
|
||||
if (item.size !== undefined) {
|
||||
const size = Number(item.size);
|
||||
if (
|
||||
!Number.isFinite(size) ||
|
||||
size < 0 ||
|
||||
size > MAX_THUMBNAIL_BYTES
|
||||
)
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = mintThumbnailKey(item.item_uid);
|
||||
const key = mintThumbnailKey();
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: key,
|
||||
ContentType: contentType,
|
||||
// Signing covers `content-length`, so a body of any other size
|
||||
// fails the signature instead of landing.
|
||||
ContentLength: size,
|
||||
});
|
||||
item.uploadUrl = await getSignedUrl(presignClient, command, {
|
||||
expiresIn: 900,
|
||||
@@ -301,14 +220,19 @@ export const handleThumbnailRead = async (
|
||||
s3Presign: S3Client;
|
||||
bucketName: string;
|
||||
bucketEndpoint: string;
|
||||
fsEntry: ThumbnailEntryStore;
|
||||
db: { write: (sql: string, params: unknown[]) => Promise<unknown> };
|
||||
},
|
||||
): Promise<void> => {
|
||||
const thumb = entry.thumbnail;
|
||||
if (typeof thumb !== 'string' || !thumb) return;
|
||||
const presignClient = deps.s3Presign;
|
||||
|
||||
if (isStoragePointer(thumb, deps.bucketEndpoint)) {
|
||||
if (
|
||||
thumb.startsWith('s3://') ||
|
||||
// Legacy format — remove after full migration
|
||||
(thumb.startsWith('https') &&
|
||||
thumb.includes(new URL(deps.bucketEndpoint).hostname))
|
||||
) {
|
||||
const key = resolveThumbnailKey(thumb);
|
||||
if (!key) {
|
||||
// Not a pointer we minted — refuse to sign it rather than hand
|
||||
@@ -322,52 +246,31 @@ export const handleThumbnailRead = async (
|
||||
{ expiresIn: 604800 },
|
||||
);
|
||||
} else if (thumb.startsWith('data')) {
|
||||
// Move an inline thumbnail into storage. The row only takes the new
|
||||
// pointer while it still holds this data URL, so of concurrent reads
|
||||
// one wins and the rest discard their upload.
|
||||
const uuid = entry.uuid ?? entry.uid;
|
||||
if (!isUuid(uuid)) return;
|
||||
// Same bar as an upload; anything else stays inline, unmigrated.
|
||||
const decoded = await decodeAndValidateThumbnail(thumb);
|
||||
if (!decoded) return;
|
||||
const key = mintThumbnailKey(uuid);
|
||||
// Inline data-URL migration: upload to S3 and update the DB entry.
|
||||
const key = mintThumbnailKey();
|
||||
const { mimeType, data } = base64ParseDataUrl(thumb);
|
||||
const newUrl = `s3://${deps.bucketName}/${key}`;
|
||||
|
||||
await deps.s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: key,
|
||||
Body: decoded.data,
|
||||
ContentType: decoded.mimeType,
|
||||
Body: data,
|
||||
ContentType: mimeType,
|
||||
}),
|
||||
);
|
||||
|
||||
let migrated: boolean;
|
||||
try {
|
||||
migrated = await deps.fsEntry.updateEntryThumbnailByUuid(
|
||||
uuid,
|
||||
`s3://${deps.bucketName}/${key}`,
|
||||
thumb,
|
||||
);
|
||||
} catch (err) {
|
||||
// The row may or may not point at the object now, so keep it.
|
||||
console.warn('[thumbnails] inline migration failed', err);
|
||||
return;
|
||||
}
|
||||
if (!migrated) {
|
||||
try {
|
||||
await deps.s3.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
Key: key,
|
||||
}),
|
||||
// Best-effort async DB update
|
||||
const uuid = entry.uuid ?? entry.uid;
|
||||
if (uuid) {
|
||||
deps.db
|
||||
.write(
|
||||
'UPDATE `fsentries` SET `thumbnail` = ? WHERE `uuid` = ?',
|
||||
[newUrl, uuid],
|
||||
)
|
||||
.catch((err: unknown) =>
|
||||
console.warn('[thumbnails] inline migration failed', err),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[thumbnails] failed to remove unused thumbnail',
|
||||
err,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
entry.thumbnail = await getSignedUrl(
|
||||
@@ -383,83 +286,63 @@ export const handleFsCopyNodeThumbnail = async (
|
||||
deps: {
|
||||
s3: S3Client;
|
||||
bucketName: string;
|
||||
bucketEndpoint: string;
|
||||
fsEntry: ThumbnailEntryStore;
|
||||
db: { write: (sql: string, params: unknown[]) => Promise<unknown> };
|
||||
},
|
||||
): Promise<void> => {
|
||||
const copy = payload.copy;
|
||||
const thumbnailUrl = copy?.thumbnail;
|
||||
if (!copy || !isUuid(copy.uuid) || typeof thumbnailUrl !== 'string') return;
|
||||
if (!isStoragePointer(thumbnailUrl, deps.bucketEndpoint)) return;
|
||||
if (!copy || !copy.uuid || typeof thumbnailUrl !== 'string') return;
|
||||
|
||||
// Same trust rule as the read and remove paths: only touch objects this
|
||||
// extension minted.
|
||||
const sourceKey = resolveThumbnailKey(thumbnailUrl);
|
||||
if (!sourceKey) return;
|
||||
|
||||
// The copied row points at the SAME object as its source. Give the copy
|
||||
// an object of its own, bound to it, so removing either entry leaves the
|
||||
// other's thumbnail alone.
|
||||
const newKey = mintThumbnailKey(copy.uuid);
|
||||
// The copied row points at the SAME S3 object as its source, and
|
||||
// fs.remove.node deletes the pointed-to object — so the first removal
|
||||
// among the sharers (an overwrite, a trash purge) would break every
|
||||
// other sharer's thumbnail. Give the copy an object of its own.
|
||||
const newKey = mintThumbnailKey();
|
||||
try {
|
||||
// Nothing larger than an upload may produce gets duplicated; such an
|
||||
// object came in through an unbounded upload URL.
|
||||
const head = await deps.s3.send(
|
||||
new HeadObjectCommand({ Bucket: deps.bucketName, Key: sourceKey }),
|
||||
);
|
||||
if (
|
||||
typeof head.ContentLength !== 'number' ||
|
||||
head.ContentLength > MAX_THUMBNAIL_BYTES
|
||||
) {
|
||||
throw new Error('thumbnail exceeds the size bound');
|
||||
}
|
||||
// Read and rewrite rather than CopyObject: a copy source skips any path
|
||||
// in the client's endpoint, so it can miss an object Bucket/Key reach.
|
||||
const source = await deps.s3.send(
|
||||
new GetObjectCommand({ Bucket: deps.bucketName, Key: sourceKey }),
|
||||
);
|
||||
// The object can be replaced after the HEAD; bound what gets buffered.
|
||||
if ((source.ContentLength ?? 0) > MAX_THUMBNAIL_BYTES) {
|
||||
(source.Body as Readable | undefined)?.destroy();
|
||||
throw new Error('thumbnail exceeds the size bound');
|
||||
}
|
||||
const body = await source.Body?.transformToByteArray();
|
||||
if (!body) throw new Error('thumbnail has no body');
|
||||
await deps.s3.send(
|
||||
new PutObjectCommand({
|
||||
new CopyObjectCommand({
|
||||
Bucket: deps.bucketName,
|
||||
CopySource: `${deps.bucketName}/${sourceKey}`,
|
||||
Key: newKey,
|
||||
Body: body,
|
||||
ContentType: source.ContentType,
|
||||
}),
|
||||
);
|
||||
} catch (err) {
|
||||
// The shared object is gone or oversized — drop the pointer rather
|
||||
// than leave the row advertising a thumbnail it doesn't have.
|
||||
await deps.fsEntry.updateEntryThumbnailByUuid(copy.uuid, null);
|
||||
// The shared object is already gone (e.g. a sharer was removed
|
||||
// before this fix existed) — the pointer is dead either way, so
|
||||
// drop it rather than leave the row advertising a thumbnail it
|
||||
// doesn't have.
|
||||
await deps.db.write(
|
||||
'UPDATE `fsentries` SET `thumbnail` = NULL WHERE `uuid` = ?',
|
||||
[copy.uuid],
|
||||
);
|
||||
console.warn('[thumbnails] failed to duplicate thumbnail on copy', err);
|
||||
return;
|
||||
}
|
||||
|
||||
await deps.fsEntry.updateEntryThumbnailByUuid(
|
||||
copy.uuid,
|
||||
`s3://${deps.bucketName}/${newKey}`,
|
||||
await deps.db.write(
|
||||
'UPDATE `fsentries` SET `thumbnail` = ? WHERE `uuid` = ?',
|
||||
[`s3://${deps.bucketName}/${newKey}`, copy.uuid],
|
||||
);
|
||||
};
|
||||
|
||||
export const handleFsRemoveNodeThumbnail = async (
|
||||
payload: { target: { thumbnail?: string | null; uuid?: string } },
|
||||
deps: { s3: S3Client; bucketName: string; bucketEndpoint: string },
|
||||
payload: { target: { thumbnail?: string | null } },
|
||||
deps: { s3: S3Client; bucketName: string },
|
||||
): Promise<void> => {
|
||||
const { thumbnail: thumbnailUrl, uuid } = payload.target;
|
||||
if (!thumbnailUrl || !isStoragePointer(thumbnailUrl, deps.bucketEndpoint))
|
||||
return;
|
||||
const thumbnailUrl = payload.target.thumbnail;
|
||||
if (!thumbnailUrl) return;
|
||||
|
||||
// The pointer decides which object gets deleted, and a key is visible to
|
||||
// anyone who can list the entry holding it. Only a key minted for this
|
||||
// entry is deleted; an unbound one may be shared or planted, so it stays.
|
||||
// Same trust rule as the read path, and load-bearing for the same reason:
|
||||
// the pointer decides which object gets deleted, so a key we didn't mint
|
||||
// would let the owner of one file destroy an object belonging to someone
|
||||
// else just by naming it here.
|
||||
const key = resolveThumbnailKey(thumbnailUrl);
|
||||
if (!key || thumbnailKeyOwner(key) !== uuid) return;
|
||||
if (!key) return;
|
||||
|
||||
await deps.s3.send(
|
||||
new DeleteObjectCommand({ Bucket: deps.bucketName, Key: key }),
|
||||
@@ -472,7 +355,6 @@ extension.on(
|
||||
await handleThumbnailCreated(event, {
|
||||
s3: getClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -499,7 +381,7 @@ extension.on('thumbnail.read', async (_key, entry: Record<string, unknown>) => {
|
||||
s3Presign: getPresignClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
fsEntry: stores.fsEntry,
|
||||
db: clients.db,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -515,8 +397,7 @@ extension.on('fs.copy.node', async (_key, payload) => {
|
||||
{
|
||||
s3: getClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
fsEntry: stores.fsEntry,
|
||||
db: clients.db,
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -528,6 +409,5 @@ extension.on('fs.remove.node', async (_key, payload) => {
|
||||
await handleFsRemoveNodeThumbnail(payload, {
|
||||
s3: getClient(),
|
||||
bucketName: thumbnailBucketName,
|
||||
bucketEndpoint: extensionBucketEndpoint,
|
||||
});
|
||||
});
|
||||
@@ -1,355 +0,0 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { Readable } from 'node:stream';
|
||||
import type { Request, Response } from 'express';
|
||||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const SECRET = 'ingress-secret';
|
||||
|
||||
// Only `extension` is stubbed; the secret comparison and the routing are the
|
||||
// real code paths, and the fake service/store layer is the boundary.
|
||||
const state = vi.hoisted(() => ({
|
||||
config: { userEmail: { secret: 'ingress-secret' } } as Record<
|
||||
string,
|
||||
unknown
|
||||
>,
|
||||
routes: [] as Array<{
|
||||
path: string;
|
||||
handler: (req: unknown, res: unknown) => unknown;
|
||||
}>,
|
||||
users: new Map<string, Record<string, unknown>>(),
|
||||
writes: [] as Array<{
|
||||
userId: number;
|
||||
path: string;
|
||||
homeRegion?: string;
|
||||
content?: unknown;
|
||||
}>,
|
||||
feedback: [] as Array<{ to: string; from: string | null; raw: Buffer }>,
|
||||
takeFeedback: true,
|
||||
}));
|
||||
|
||||
vi.mock('@heyputer/backend/src/extensions', () => ({
|
||||
extension: {
|
||||
get config() {
|
||||
return state.config;
|
||||
},
|
||||
post(
|
||||
path: string,
|
||||
_opts: unknown,
|
||||
handler: (req: unknown, res: unknown) => unknown,
|
||||
) {
|
||||
state.routes.push({ path, handler });
|
||||
},
|
||||
import(layer: string) {
|
||||
if (layer === 'client') {
|
||||
return {
|
||||
event: {
|
||||
async emitAndWait(
|
||||
key: string,
|
||||
data: {
|
||||
to: string;
|
||||
from: string | null;
|
||||
raw: Buffer;
|
||||
handled: boolean;
|
||||
},
|
||||
) {
|
||||
if (key !== 'email.ingress.feedback') return;
|
||||
state.feedback.push(data);
|
||||
if (state.takeFeedback) data.handled = true;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (layer === 'store') {
|
||||
return {
|
||||
user: {
|
||||
getByUsername: async (username: string) =>
|
||||
state.users.get(username) ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
fs: {
|
||||
async write(
|
||||
userId: number,
|
||||
{
|
||||
fileMetadata,
|
||||
fileContent,
|
||||
}: {
|
||||
fileMetadata: { path: string };
|
||||
fileContent: unknown;
|
||||
},
|
||||
_uploadTracker?: unknown,
|
||||
_storageAllowanceMax?: number,
|
||||
homeRegion?: string,
|
||||
) {
|
||||
state.writes.push({
|
||||
userId,
|
||||
path: fileMetadata.path,
|
||||
homeRegion,
|
||||
content: fileContent,
|
||||
});
|
||||
return { fsEntry: { path: fileMetadata.path } };
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
import './index.js';
|
||||
|
||||
const permanent = (id: number, username: string) => ({
|
||||
id,
|
||||
username,
|
||||
email: `${username}@example.com`,
|
||||
password: '$2b$10$hash',
|
||||
});
|
||||
/** No password and no email is what makes an account temporary. */
|
||||
const temp = (id: number, username: string) => ({
|
||||
id,
|
||||
username,
|
||||
email: null,
|
||||
password: null,
|
||||
});
|
||||
|
||||
const ingress = async (
|
||||
to: string,
|
||||
over: { secret?: string; contentLength?: string | null } = {},
|
||||
) => {
|
||||
const route = state.routes.find((r) => r.path === '/email/ingress');
|
||||
if (!route) throw new Error('route not registered');
|
||||
|
||||
const body = Buffer.from('From: [email protected]\r\n\r\nhello\r\n');
|
||||
const req = Readable.from([body], { objectMode: false }) as unknown as {
|
||||
headers: Record<string, string>;
|
||||
query: Record<string, string>;
|
||||
};
|
||||
// `contentLength: null` sends no length header at all, which is the one
|
||||
// framing the HTTP parser does not bound for us.
|
||||
req.headers =
|
||||
over.contentLength === null
|
||||
? {}
|
||||
: {
|
||||
'content-length': over.contentLength ?? String(body.length),
|
||||
};
|
||||
req.query = {
|
||||
SECRET: over.secret ?? SECRET,
|
||||
to,
|
||||
from: '[email protected]',
|
||||
subject: 'hi',
|
||||
};
|
||||
|
||||
let status = 200;
|
||||
let ended = false;
|
||||
// `end` takes (chunk, encoding) and no callback, the way the compression
|
||||
// middleware leaves it, and emits `finish` — which is what the refusal
|
||||
// paths hang their drain/destroy on. A stub that accepts a callback
|
||||
// instead hides the bug that took the 413 path to production as a 500.
|
||||
const emitter = new EventEmitter();
|
||||
const res = {
|
||||
status(code: number) {
|
||||
status = code;
|
||||
return res;
|
||||
},
|
||||
end(chunk?: unknown, encoding?: BufferEncoding) {
|
||||
if (chunk) Buffer.byteLength(chunk as string, encoding);
|
||||
ended = true;
|
||||
emitter.emit('finish');
|
||||
},
|
||||
once(event: string, listener: () => void) {
|
||||
emitter.once(event, listener);
|
||||
return res;
|
||||
},
|
||||
} as unknown as Response;
|
||||
|
||||
await route.handler(req as unknown as Request, res);
|
||||
return {
|
||||
status,
|
||||
ended,
|
||||
drained: (req as unknown as Readable).readableFlowing,
|
||||
destroyed: (req as unknown as Readable).destroyed,
|
||||
};
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
state.users.clear();
|
||||
state.writes.length = 0;
|
||||
state.feedback.length = 0;
|
||||
state.takeFeedback = true;
|
||||
});
|
||||
|
||||
describe('temporary accounts cannot receive', () => {
|
||||
test('mail for a temp account is accepted and dropped', async () => {
|
||||
state.users.set('tempy', temp(1, 'tempy'));
|
||||
|
||||
const { status, ended, drained } = await ingress('[email protected]');
|
||||
// Accepted, so the relay neither retries nor bounces to the sender.
|
||||
expect(status).toBe(200);
|
||||
expect(ended).toBe(true);
|
||||
// The body is read off the wire rather than left unconsumed.
|
||||
expect(drained).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('mail for a permanent account still lands in the mailbox', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(200);
|
||||
expect(state.writes).toHaveLength(1);
|
||||
expect(state.writes[0].userId).toBe(2);
|
||||
expect(state.writes[0].path).toMatch(/^\/bob\/\.mail\/objects\//);
|
||||
});
|
||||
|
||||
test('an unknown address is still rejected outright', async () => {
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(404);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('inbound mail is placed in the recipient home region', () => {
|
||||
/** A permanent account with explicit placement columns. */
|
||||
const placed = (
|
||||
id: number,
|
||||
username: string,
|
||||
over: { home?: string | null; signup_server?: string | null },
|
||||
) => ({
|
||||
...permanent(id, username),
|
||||
home: null,
|
||||
signup_server: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
test("uses the recipient's own home region when they have one", async () => {
|
||||
state.users.set('bob', placed(2, 'bob', { home: 'frankfurt' }));
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('frankfurt');
|
||||
});
|
||||
|
||||
test('falls back to the server that served their signup', async () => {
|
||||
// Every account that predates the `home` column is this case.
|
||||
state.users.set(
|
||||
'bob',
|
||||
placed(2, 'bob', { home: null, signup_server: 'london' }),
|
||||
);
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('london');
|
||||
});
|
||||
|
||||
test('falls back to the primary region when neither is set', async () => {
|
||||
state.users.set('bob', placed(2, 'bob', {}));
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('oregon');
|
||||
});
|
||||
|
||||
test('home wins over signup_server when both are set', async () => {
|
||||
state.users.set(
|
||||
'bob',
|
||||
placed(2, 'bob', { home: 'sydney', signup_server: 'london' }),
|
||||
);
|
||||
|
||||
await ingress('[email protected]');
|
||||
expect(state.writes[0].homeRegion).toBe('sydney');
|
||||
});
|
||||
});
|
||||
|
||||
describe('refusals dispose of the body nobody read', () => {
|
||||
test('an unknown recipient drains it and keeps the connection', async () => {
|
||||
// The secret checked out and the length is under the cap, so the body
|
||||
// was ours to read: discard it rather than resetting a usable socket.
|
||||
const { status, drained, destroyed } =
|
||||
await ingress('[email protected]');
|
||||
expect(status).toBe(404);
|
||||
expect(drained).toBe(true);
|
||||
expect(destroyed).toBe(false);
|
||||
});
|
||||
|
||||
test('a bad secret closes rather than reading the upload', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
secret: 'not-the-secret',
|
||||
});
|
||||
expect(status).toBe(403);
|
||||
// Nothing authenticated this caller; their bytes are not worth reading.
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('no content-length is refused with 411', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
contentLength: null,
|
||||
});
|
||||
expect(status).toBe(411);
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('a declared length over the cap is refused with 413', async () => {
|
||||
state.users.set('bob', permanent(2, 'bob'));
|
||||
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
contentLength: String(25 * 1024 * 1024 + 1),
|
||||
});
|
||||
expect(status).toBe(413);
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('feedback addresses', () => {
|
||||
test('are handed to a listener and reach no mailbox', async () => {
|
||||
const { status, ended } = await ingress('[email protected]');
|
||||
expect(status).toBe(200);
|
||||
expect(ended).toBe(true);
|
||||
expect(state.feedback).toHaveLength(1);
|
||||
expect(state.feedback[0].to).toBe('[email protected]');
|
||||
expect(state.feedback[0].from).toBe('[email protected]');
|
||||
expect(state.feedback[0].raw.toString()).toContain('hello');
|
||||
expect(state.writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('match the local part case-insensitively', async () => {
|
||||
await ingress('[email protected]');
|
||||
expect(state.feedback).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('fall through to normal routing when nobody takes them', async () => {
|
||||
state.takeFeedback = false;
|
||||
state.users.set('abuse', permanent(3, 'abuse'));
|
||||
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(200);
|
||||
expect(state.feedback).toHaveLength(1);
|
||||
// The body was already read, so the buffered copy is what is stored.
|
||||
expect(state.writes).toHaveLength(1);
|
||||
expect(Buffer.isBuffer(state.writes[0].content)).toBe(true);
|
||||
});
|
||||
|
||||
test('an unclaimed one with no such account is still a 404', async () => {
|
||||
state.takeFeedback = false;
|
||||
const { status } = await ingress('[email protected]');
|
||||
expect(status).toBe(404);
|
||||
});
|
||||
|
||||
test('only count on a Puter mail domain', async () => {
|
||||
state.users.set('abuse', permanent(3, 'abuse'));
|
||||
await ingress('[email protected]');
|
||||
expect(state.feedback).toEqual([]);
|
||||
});
|
||||
|
||||
test('are refused over the feedback size cap without being read', async () => {
|
||||
const { status, destroyed } = await ingress('[email protected]', {
|
||||
contentLength: String(5 * 1024 * 1024 + 1),
|
||||
});
|
||||
expect(status).toBe(413);
|
||||
expect(destroyed).toBe(true);
|
||||
expect(state.feedback).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,161 +0,0 @@
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import type { Request } from 'express';
|
||||
import {
|
||||
MAX_MESSAGE_BYTES,
|
||||
isPuterEmailAddress,
|
||||
isTempUser,
|
||||
puterEmailUsername,
|
||||
storeInboxMessage,
|
||||
} from '@heyputer/backend/src/services/email/mailbox.js';
|
||||
|
||||
// copied form peer secrets checker
|
||||
const COMPARE_KEY = randomBytes(32);
|
||||
const secretsEqual = (a: string, b: string): boolean =>
|
||||
timingSafeEqual(
|
||||
createHmac('sha256', COMPARE_KEY).update(a).digest(),
|
||||
createHmac('sha256', COMPARE_KEY).update(b).digest(),
|
||||
);
|
||||
|
||||
const USER_EMAIL_CONFIG = (extension.config as Record<string, unknown>)
|
||||
.userEmail as { secret?: string; feedbackAddresses?: unknown } | undefined;
|
||||
|
||||
const INGRESS_SECRET = USER_EMAIL_CONFIG?.secret;
|
||||
|
||||
/**
|
||||
* Local parts that take complaint feedback rather than naming a mailbox. The
|
||||
* same names are reserved at signup, so no account can be handed them.
|
||||
*/
|
||||
const DEFAULT_FEEDBACK_ADDRESSES = ['fbl', 'abuse', 'postmaster'];
|
||||
const FEEDBACK_ADDRESSES = new Set(
|
||||
(Array.isArray(USER_EMAIL_CONFIG?.feedbackAddresses)
|
||||
? USER_EMAIL_CONFIG.feedbackAddresses
|
||||
: DEFAULT_FEEDBACK_ADDRESSES
|
||||
).map((local) => String(local).toLowerCase()),
|
||||
);
|
||||
|
||||
/**
|
||||
* Feedback is handed over whole, so it is read into memory; a complaint report
|
||||
* quotes one message and has no business near the mailbox ceiling.
|
||||
*/
|
||||
const MAX_FEEDBACK_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
const isFeedbackAddress = (to: string): boolean =>
|
||||
isPuterEmailAddress(to) &&
|
||||
FEEDBACK_ADDRESSES.has(puterEmailUsername(to).toLowerCase());
|
||||
|
||||
const readBody = async (req: Request): Promise<Buffer> => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) chunks.push(chunk as Buffer);
|
||||
return Buffer.concat(chunks);
|
||||
};
|
||||
|
||||
if (!INGRESS_SECRET) {
|
||||
console.warn(
|
||||
'[useremailrx] config.userEmail.secret unset - email ingress disabled',
|
||||
);
|
||||
} else {
|
||||
extension.post(
|
||||
'/email/ingress',
|
||||
{
|
||||
subdomain: 'api',
|
||||
},
|
||||
async (req, res) => {
|
||||
// Answer first, then deal with the body nobody read. Touching the
|
||||
// request before the response has flushed takes the response down
|
||||
// with the connection and the caller sees a reset with no status,
|
||||
// so this hangs off `finish` — and it has to be the event rather
|
||||
// than `res.end(cb)`, because the compression middleware replaces
|
||||
// `res.end` with an `(chunk, encoding)` signature that has no
|
||||
// callback parameter, so a callback lands in `chunk` and dies in
|
||||
// `Buffer.byteLength`.
|
||||
//
|
||||
// `drain` discards the rest of a body we were entitled to read and
|
||||
// leaves the connection reusable. `close` is for bodies we do not
|
||||
// want to read at all: an unauthenticated caller's, or one whose
|
||||
// declared length is missing or over the cap.
|
||||
const refuse = (status: number, how: 'drain' | 'close') => {
|
||||
res.once('finish', () =>
|
||||
how === 'drain' ? req.resume() : req.destroy(),
|
||||
);
|
||||
res.status(status).end();
|
||||
};
|
||||
|
||||
const presented = req.query.SECRET;
|
||||
if (
|
||||
typeof presented !== 'string' ||
|
||||
!secretsEqual(presented, INGRESS_SECRET)
|
||||
) {
|
||||
// Nothing has authenticated this caller, so refuse to spend
|
||||
// bandwidth reading whatever they were uploading.
|
||||
refuse(403, 'close');
|
||||
return;
|
||||
}
|
||||
|
||||
const size = Number(req.headers['content-length']);
|
||||
if (!Number.isInteger(size) || size <= 0) {
|
||||
refuse(411, 'close');
|
||||
return;
|
||||
}
|
||||
if (size > MAX_MESSAGE_BYTES) {
|
||||
refuse(413, 'close');
|
||||
return;
|
||||
}
|
||||
|
||||
const stores = extension.import('store');
|
||||
const services = extension.import('service');
|
||||
|
||||
const to = String(req.query.to ?? '');
|
||||
let content: Buffer | Request = req;
|
||||
if (isFeedbackAddress(to)) {
|
||||
if (size > MAX_FEEDBACK_BYTES) {
|
||||
refuse(413, 'close');
|
||||
return;
|
||||
}
|
||||
const raw = await readBody(req);
|
||||
const event = {
|
||||
to,
|
||||
from:
|
||||
typeof req.query.from === 'string'
|
||||
? req.query.from
|
||||
: null,
|
||||
raw,
|
||||
handled: false,
|
||||
};
|
||||
await extension
|
||||
.import('client')
|
||||
.event.emitAndWait('email.ingress.feedback', event, {});
|
||||
if (event.handled) {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
// Nobody took it: route it like any other message.
|
||||
content = raw;
|
||||
}
|
||||
|
||||
const user = await stores.user.getByUsername(
|
||||
puterEmailUsername(to),
|
||||
);
|
||||
if (!user) {
|
||||
refuse(404, 'drain');
|
||||
return;
|
||||
}
|
||||
|
||||
if (isTempUser(user)) {
|
||||
if (content === req) req.resume();
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
await storeInboxMessage(services.fs, user, {
|
||||
subject: req.query.subject,
|
||||
content,
|
||||
size: Buffer.isBuffer(content) ? content.length : size,
|
||||
});
|
||||
|
||||
const { SECRET: _secret, ..._loggableQuery } = req.query;
|
||||
// console.log('got email: ', puterUser, loggableQuery, size);
|
||||
res.end();
|
||||
},
|
||||
);
|
||||
}
|
||||
Generated
-12
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"name": "emailrx",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "emailrx",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"name": "emailrx",
|
||||
"version": "1.0.0",
|
||||
"main": "index.ts",
|
||||
"description": "Internal service which handles routing puter emails to users"
|
||||
}
|
||||
+26
-187
@@ -1,8 +1,14 @@
|
||||
import type { Request, Response } from 'express';
|
||||
import { installSeatStub } from '@heyputer/backend/testFixtures/seatStub';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { makeActor, type Actor } from '../src/backend/core/actor.ts';
|
||||
import {
|
||||
afterAll,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { makeActor } from '../src/backend/core/actor.ts';
|
||||
import { runWithContext } from '../src/backend/core/context.ts';
|
||||
import { configContainer } from '../src/backend/exports.ts';
|
||||
import { PuterServer } from '../src/backend/server.ts';
|
||||
@@ -43,6 +49,7 @@ beforeAll(async () => {
|
||||
create_shortcut: true,
|
||||
payment_bypass: true,
|
||||
},
|
||||
teams_enabled: true,
|
||||
} as never);
|
||||
});
|
||||
|
||||
@@ -76,20 +83,21 @@ describe('whoami extension — handleWhoami', () => {
|
||||
};
|
||||
|
||||
const seatOf = async (teamName: string) => {
|
||||
const owner = await seedUser();
|
||||
const seat = await seedSeat();
|
||||
const team = {
|
||||
uid: `wt-${Math.random().toString(36).slice(2, 9)}`,
|
||||
const team = await server.stores.team.create({
|
||||
ownerUserId: owner.id as number,
|
||||
name: teamName,
|
||||
};
|
||||
installSeatStub(server as never, [seat.id as number], {
|
||||
teamUid: team.uid,
|
||||
teamName: team.name,
|
||||
handle: `wt-${Math.random().toString(36).slice(2, 9)}`,
|
||||
});
|
||||
await server.stores.team.addMember(team.uid, seat.id as number, {
|
||||
orgOwned: true,
|
||||
});
|
||||
return { seat, team };
|
||||
};
|
||||
|
||||
it('names no team where teams is off', async () => {
|
||||
const { seat } = await seatOf('Acme Corp');
|
||||
it('names the team for a seat', async () => {
|
||||
const { seat, team } = await seatOf('Acme Corp');
|
||||
const { res, captured } = makeRes();
|
||||
|
||||
await runWithContext(
|
||||
@@ -97,8 +105,10 @@ describe('whoami extension — handleWhoami', () => {
|
||||
() => handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
// Off, so the seat is never looked up.
|
||||
expect((captured.body as { team?: unknown }).team).toBeUndefined();
|
||||
expect((captured.body as { team?: unknown }).team).toEqual({
|
||||
uid: team.uid,
|
||||
name: 'Acme Corp',
|
||||
});
|
||||
});
|
||||
|
||||
it('says nothing for an account that is not a seat', async () => {
|
||||
@@ -160,81 +170,6 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(captured.body).toEqual({ error: 'User not found' });
|
||||
});
|
||||
|
||||
describe("an app's view of the email", () => {
|
||||
const appActorFor = async (user: { uuid: string; id: unknown }) => {
|
||||
const app = await (
|
||||
server.stores.app.create as unknown as (
|
||||
fields: Record<string, unknown>,
|
||||
opts: { ownerUserId: number },
|
||||
) => Promise<{ uid: string; id: number }>
|
||||
)(
|
||||
{
|
||||
name: `we-${uuidv4()}`,
|
||||
title: 'Email reader',
|
||||
index_url: `https://${uuidv4()}.example/`,
|
||||
},
|
||||
{ ownerUserId: user.id as number },
|
||||
);
|
||||
return {
|
||||
app,
|
||||
actor: makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: app.uid, id: app.id },
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
const whoamiAs = async (actor: Actor) => {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () =>
|
||||
handleWhoami(makeReq(), res),
|
||||
);
|
||||
return captured;
|
||||
};
|
||||
|
||||
it('is withheld until the user grants it, then shown', async () => {
|
||||
const user = await seedUser();
|
||||
const { app, actor } = await appActorFor(user);
|
||||
|
||||
const before = (await whoamiAs(actor)).body as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
expect(before).not.toHaveProperty('email');
|
||||
expect(before).not.toHaveProperty('unconfirmed_email');
|
||||
|
||||
await server.services.permission.grantUserAppPermission(
|
||||
makeActor({ user: { uuid: user.uuid, id: user.id as number } }),
|
||||
app.uid,
|
||||
`user:${user.uuid}:email:read`,
|
||||
);
|
||||
const after = (await whoamiAs(actor)).body as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
expect(after.email).toBe(user.email);
|
||||
expect(after.unconfirmed_email).toBe(user.email);
|
||||
});
|
||||
|
||||
it('answers 404 for a missing account even when the check fails', async () => {
|
||||
const check = vi
|
||||
.spyOn(server.services.permission, 'check')
|
||||
.mockRejectedValue(new Error('scan failed'));
|
||||
try {
|
||||
const captured = await whoamiAs(
|
||||
makeActor({
|
||||
user: { uuid: 'ghost-uuid', id: 99_999_998 },
|
||||
app: { uid: 'app-ghost', id: 1 },
|
||||
}),
|
||||
);
|
||||
expect(captured.statusCode).toBe(404);
|
||||
expect(captured.body).toEqual({ error: 'User not found' });
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('returns full user details for a user actor', async () => {
|
||||
const user = await seedUser();
|
||||
const { res, captured } = makeRes();
|
||||
@@ -350,104 +285,6 @@ describe('whoami extension — handleWhoami', () => {
|
||||
expect(body.directories).toBeUndefined();
|
||||
});
|
||||
|
||||
describe('fields added by whoami.details listeners', () => {
|
||||
const addAccountDetails = (
|
||||
_key: unknown,
|
||||
event: { details: Record<string, unknown> },
|
||||
) => {
|
||||
Object.assign(event.details, {
|
||||
subscribed: true,
|
||||
paid_storage: 1024,
|
||||
subscription: { tier: 'pro', status: 'trialing' },
|
||||
accountEligible: true,
|
||||
});
|
||||
};
|
||||
|
||||
const whoamiWithListener = async (
|
||||
actor: Actor,
|
||||
listener: typeof addAccountDetails = addAccountDetails,
|
||||
) => {
|
||||
const { res, captured } = makeRes();
|
||||
server.clients.event.on('whoami.details', listener);
|
||||
try {
|
||||
await runWithContext({ actor }, () =>
|
||||
handleWhoami(makeReq(), res),
|
||||
);
|
||||
} finally {
|
||||
server.clients.event.off('whoami.details', listener);
|
||||
}
|
||||
return captured.body as Record<string, unknown>;
|
||||
};
|
||||
|
||||
it('reach an app actor only as the documented subscribed / paid_storage', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(body.subscribed).toBe(true);
|
||||
expect(body.paid_storage).toBe(1024);
|
||||
expect(body).not.toHaveProperty('subscription');
|
||||
expect(body).not.toHaveProperty('accountEligible');
|
||||
// Fields the handler built itself still ship.
|
||||
expect(body.username).toBe(user.username);
|
||||
expect(body.app_name).toBe('app-test-actor');
|
||||
});
|
||||
|
||||
it('are withheld from an app reached through an access token it issued', async () => {
|
||||
const user = await seedUser();
|
||||
const owner = { uuid: user.uuid, id: user.id as number };
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
user: owner,
|
||||
accessToken: {
|
||||
uid: 'tok-1',
|
||||
issuer: makeActor({
|
||||
user: owner,
|
||||
app: { uid: 'app-issuer' },
|
||||
}),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(body.subscribed).toBe(true);
|
||||
expect(body).not.toHaveProperty('subscription');
|
||||
expect(body).not.toHaveProperty('accountEligible');
|
||||
});
|
||||
|
||||
it('all reach a user actor', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({ user: { uuid: user.uuid, id: user.id as number } }),
|
||||
);
|
||||
|
||||
expect(body).toMatchObject({
|
||||
subscribed: true,
|
||||
paid_storage: 1024,
|
||||
subscription: { tier: 'pro', status: 'trialing' },
|
||||
accountEligible: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('cannot add taskbar_items for an app actor, which core omits', async () => {
|
||||
const user = await seedUser();
|
||||
const body = await whoamiWithListener(
|
||||
makeActor({
|
||||
user: { uuid: user.uuid, id: user.id as number },
|
||||
app: { uid: 'app-test-actor' },
|
||||
}),
|
||||
(_key, event) => {
|
||||
event.details.taskbar_items = [{ name: 'injected' }];
|
||||
},
|
||||
);
|
||||
|
||||
expect(body).not.toHaveProperty('taskbar_items');
|
||||
});
|
||||
});
|
||||
|
||||
it('redacts tmp_password from metadata for user actors', async () => {
|
||||
const user = await seedUser();
|
||||
await server.stores.user.updateMetadata(user.id as number, {
|
||||
@@ -516,7 +353,9 @@ describe('whoami extension — handleWhoami', () => {
|
||||
}),
|
||||
]) {
|
||||
const { res, captured } = makeRes();
|
||||
await runWithContext({ actor }, () => handleWhoami(makeReq(), res));
|
||||
await runWithContext({ actor }, () =>
|
||||
handleWhoami(makeReq(), res),
|
||||
);
|
||||
|
||||
const body = captured.body as Record<string, unknown>;
|
||||
expect(body.phone).toBeUndefined();
|
||||
|
||||
+229
-48
@@ -1,10 +1,11 @@
|
||||
import { Context } from '@heyputer/backend/src/core';
|
||||
import { extension } from '@heyputer/backend/src/extensions';
|
||||
import { APP_ICON_SIZES } from '@heyputer/backend/src/util/appIcon.js';
|
||||
import {
|
||||
buildUserDetails,
|
||||
scrubSensitive,
|
||||
} from '@heyputer/backend/src/util/userDetails.js';
|
||||
cardFallbackDepsFrom,
|
||||
isCardFallbackEligible,
|
||||
} from '@heyputer/backend/src/util/cardFallback.js';
|
||||
import { APP_ICON_SIZES } from '@heyputer/backend/src/util/appIcon.js';
|
||||
import { getTaskbarItems } from '@heyputer/backend/src/util/taskbarItems.js';
|
||||
import type { Request, Response } from 'express';
|
||||
import TimeAgo from 'javascript-time-ago';
|
||||
import localeEn from 'javascript-time-ago/locale/en';
|
||||
@@ -18,13 +19,87 @@ const timeago = (() => {
|
||||
return new TimeAgo('en-US');
|
||||
})();
|
||||
|
||||
// What an app actor may receive from `whoami.details` listeners. Anything
|
||||
// else a listener adds describes the account, and only ships to user actors.
|
||||
const APP_VISIBLE_LISTENER_KEYS: ReadonlySet<string> = new Set([
|
||||
'subscribed',
|
||||
'paid_storage',
|
||||
// User timestamps come off the DB as SQL datetime strings; the wire format
|
||||
// for all of them is unix seconds. Unparseable values are dropped rather
|
||||
// than sent as NaN.
|
||||
const toUnixSeconds = (value: unknown): number | undefined => {
|
||||
if (!value) return undefined;
|
||||
const ms = new Date(value as string | number | Date).getTime();
|
||||
return Number.isNaN(ms) ? undefined : Math.round(ms / 1000);
|
||||
};
|
||||
|
||||
// Allowlist of `config.feature_flags` keys safe to surface via /whoami.
|
||||
// Anything not listed here stays server-side, so internal flags
|
||||
// (payment_bypass, staff_only_*, etc.) cannot leak by accident. Add a
|
||||
// flag here when, and only when, the client actually needs to read it.
|
||||
const CLIENT_VISIBLE_FEATURE_FLAGS: ReadonlySet<string> = new Set([
|
||||
'create_shortcut',
|
||||
'download_directory',
|
||||
'prompt_user_when_navigation_away_from_puter',
|
||||
]);
|
||||
|
||||
// Keys that must never leave the server, whoever put them on the response.
|
||||
// `details` is an explicit pick, but the `whoami.details` event hands
|
||||
// listeners the full UserRow next to the object they may write to, and
|
||||
// `metadata` is a free-form blob — so any of these can arrive on the
|
||||
// response without an edit to the pick above. The scrub runs last, over the
|
||||
// whole payload, and is the one place that decides what "sensitive" means.
|
||||
//
|
||||
// Credentials and single-use tokens, the payment/phone identifiers used for
|
||||
// verification (`card_fingerprint` is the Stripe fingerprint, stable per
|
||||
// card number), the network identity recorded at signup, and internal
|
||||
// anti-abuse bookkeeping. `requires_phone_verification` /
|
||||
// `requires_card_verification` stay: they are the flags the GUI acts on, and
|
||||
// they carry no identifier.
|
||||
const SENSITIVE_KEYS: ReadonlySet<string> = new Set([
|
||||
'password',
|
||||
'tmp_password',
|
||||
'pass_recovery_token',
|
||||
'email_confirm_code',
|
||||
'email_confirm_token',
|
||||
'change_email_confirm_token',
|
||||
'otp_secret',
|
||||
'otp_recovery_codes',
|
||||
'card_fingerprint',
|
||||
'phone',
|
||||
'clean_email',
|
||||
'signup_ip',
|
||||
'signup_ip_forwarded',
|
||||
'signup_user_agent',
|
||||
'signup_origin',
|
||||
'signup_server',
|
||||
'audit_metadata',
|
||||
]);
|
||||
|
||||
// Depth-limited, cycle-safe walk deleting every SENSITIVE_KEYS entry it finds
|
||||
// at any level (`metadata` and `taskbar_items` are both nested structures).
|
||||
const scrubSensitive = (
|
||||
value: unknown,
|
||||
seen: Set<object> = new Set(),
|
||||
depth = 0,
|
||||
): void => {
|
||||
if (depth > 8 || value === null || typeof value !== 'object') return;
|
||||
if (seen.has(value as object)) return;
|
||||
seen.add(value as object);
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const entry of value) scrubSensitive(entry, seen, depth + 1);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const key of Object.keys(value as Record<string, unknown>)) {
|
||||
if (SENSITIVE_KEYS.has(key)) {
|
||||
delete (value as Record<string, unknown>)[key];
|
||||
continue;
|
||||
}
|
||||
scrubSensitive(
|
||||
(value as Record<string, unknown>)[key],
|
||||
seen,
|
||||
depth + 1,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const handleWhoami = async (
|
||||
req: Request,
|
||||
res: Response,
|
||||
@@ -36,24 +111,13 @@ export const handleWhoami = async (
|
||||
}
|
||||
|
||||
const isUser = !actor.effectiveApp;
|
||||
const canReadEmail = async (uuid: unknown) => {
|
||||
try {
|
||||
return await services.permission.check(
|
||||
actor,
|
||||
`user:${uuid}:email:read`,
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
// Started beside the row read: the actor carries the row's uuid.
|
||||
const emailCheck = isUser ? null : canReadEmail(actor.user.uuid);
|
||||
const user = await stores.user.getById(actor.user.id);
|
||||
if (!user) {
|
||||
res.status(404).json({ error: 'User not found' });
|
||||
return;
|
||||
}
|
||||
|
||||
const oidcOnly = user.password === null;
|
||||
const rawIconSize =
|
||||
typeof req.query?.icon_size === 'string'
|
||||
? Number(req.query.icon_size)
|
||||
@@ -62,22 +126,153 @@ export const handleWhoami = async (
|
||||
rawIconSize !== undefined && APP_ICON_SIZES.includes(rawIconSize)
|
||||
? rawIconSize
|
||||
: undefined;
|
||||
const noIcons = !iconSize;
|
||||
|
||||
const details = await buildUserDetails(
|
||||
user,
|
||||
{ config: extension.config, clients, stores, services },
|
||||
{ isUser, iconSize, noIcons: !iconSize },
|
||||
);
|
||||
details.human_readable_age = user.timestamp
|
||||
? timeago.format(new Date(user.timestamp as string))
|
||||
: null;
|
||||
// Feature flags come from `config.feature_flags`. We only forward keys
|
||||
// listed in CLIENT_VISIBLE_FEATURE_FLAGS so internal flags can't leak.
|
||||
// Non-boolean values (e.g. `"true"` as a string) are coerced so the
|
||||
// client never has to guess.
|
||||
const rawFlags = extension.config.feature_flags ?? {};
|
||||
const feature_flags: Record<string, boolean> = {};
|
||||
for (const [k, v] of Object.entries(rawFlags)) {
|
||||
if (CLIENT_VISIBLE_FEATURE_FLAGS.has(k)) {
|
||||
feature_flags[k] = Boolean(v);
|
||||
}
|
||||
}
|
||||
|
||||
// Deep-copied (it is decoded JSON) so the scrub below edits the response
|
||||
// and not the cached UserRow. Sensitive keys inside it — tmp_password and
|
||||
// anything else on the denylist — are removed by scrubSensitive.
|
||||
const metadata = user.metadata
|
||||
? structuredClone(user.metadata)
|
||||
: user.metadata;
|
||||
|
||||
const details: Record<string, unknown> = {
|
||||
username: user.username,
|
||||
uuid: user.uuid,
|
||||
email: user.email,
|
||||
unconfirmed_email: user.email,
|
||||
email_confirmed: user.email_confirmed || user.username === 'admin',
|
||||
requires_email_confirmation: user.requires_email_confirmation,
|
||||
// The phone number itself is deliberately absent: nothing on the
|
||||
// client reads it, and it is PII that would otherwise be handed to
|
||||
// every app actor. Only the verification flag ships.
|
||||
requires_phone_verification: user.requires_phone_verification,
|
||||
requires_card_verification: user.requires_card_verification,
|
||||
// A seat reaches nothing until it replaces its admin's password.
|
||||
requires_password_change: user.requires_password_change,
|
||||
// The SMS-to-card escape hatch: true once this user is out of SMS send
|
||||
// attempts and may verify a card instead. It has to ship from here
|
||||
// because /send-confirm-phone can no longer say so — by the time the
|
||||
// fallback opens, further sends are rejected by that route's own rate
|
||||
// limit before any handler runs, so a page reload would otherwise lose
|
||||
// an offer that stays valid for 24 hours. Only for user actors, and it
|
||||
// costs no KV read unless the account is actually phone-gated.
|
||||
card_fallback_available: isUser
|
||||
? await isCardFallbackEligible(
|
||||
extension.config,
|
||||
user,
|
||||
async (key) => (await stores.kv.get({ key })).res,
|
||||
cardFallbackDepsFrom(clients),
|
||||
)
|
||||
: false,
|
||||
desktop_bg_url: user.desktop_bg_url,
|
||||
desktop_bg_color: user.desktop_bg_color,
|
||||
desktop_bg_fit: user.desktop_bg_fit,
|
||||
is_temp: user.password === null && user.email === null,
|
||||
is_user_token: true,
|
||||
// Present only once the account has actually asked for a code (see the
|
||||
// referral extension) — null until then, and never minted from here:
|
||||
// this endpoint is polled, and a mint is a write.
|
||||
referral_code: user.referral_code,
|
||||
oidc_only: oidcOnly,
|
||||
taskbar_items: isUser
|
||||
? await getTaskbarItems(
|
||||
user,
|
||||
{
|
||||
clients,
|
||||
stores,
|
||||
services,
|
||||
apiBaseUrl: String(extension.config.api_base_url ?? ''),
|
||||
config: extension.config,
|
||||
},
|
||||
{ iconSize, noIcons },
|
||||
)
|
||||
: undefined,
|
||||
otp: !!user.otp_enabled,
|
||||
feature_flags,
|
||||
created_ts: toUnixSeconds(user.timestamp),
|
||||
human_readable_age: user.timestamp
|
||||
? timeago.format(new Date(user.timestamp as string))
|
||||
: null,
|
||||
metadata,
|
||||
hasDevAccountAccess: !!user.metadata?.hasDevAccountAccess,
|
||||
};
|
||||
|
||||
// OIDC revalidate URL for password-less accounts
|
||||
if (oidcOnly) {
|
||||
try {
|
||||
const provider = await services.oidc.getLinkedProviderForUser(
|
||||
user.id as number,
|
||||
);
|
||||
if (provider) {
|
||||
const origin = (extension.config.origin ?? '').replace(
|
||||
/\/$/,
|
||||
'',
|
||||
);
|
||||
details.oidc_revalidate_url = `${origin}/auth/oidc/${provider}/start?flow=revalidate&user_uuid=${encodeURIComponent(user.uuid)}`;
|
||||
}
|
||||
} catch {
|
||||
// OIDC not configured
|
||||
}
|
||||
}
|
||||
|
||||
// Directories — only sent to user actors
|
||||
if (isUser) {
|
||||
const directories: Record<string, unknown> = {};
|
||||
const nameToProp: Record<string, string> = {
|
||||
desktop_uuid: `/${user.username}/Desktop`,
|
||||
appdata_uuid: `/${user.username}/AppData`,
|
||||
documents_uuid: `/${user.username}/Documents`,
|
||||
pictures_uuid: `/${user.username}/Pictures`,
|
||||
videos_uuid: `/${user.username}/Videos`,
|
||||
trash_uuid: `/${user.username}/Trash`,
|
||||
};
|
||||
for (const k in nameToProp) {
|
||||
directories[nameToProp[k]] = user[k];
|
||||
}
|
||||
details.directories = directories;
|
||||
}
|
||||
|
||||
// The team an account belongs to, when it is one a team pays for. User
|
||||
// actors only, and only where teams are on.
|
||||
if (isUser && extension.config.teams_enabled === true) {
|
||||
try {
|
||||
const seat = await stores.team.getOrgSeat(user.id);
|
||||
if (seat) {
|
||||
details.team = {
|
||||
uid: seat.team_uid,
|
||||
name: seat.team_name ?? null,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
// Never fail whoami over this; the account still works without it.
|
||||
console.warn('[whoami] team lookup failed:', (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// Last activity
|
||||
const lastActivityTs = toUnixSeconds(user.last_activity_ts);
|
||||
if (lastActivityTs !== undefined) {
|
||||
details.last_activity_ts = lastActivityTs;
|
||||
}
|
||||
|
||||
// Strip sensitive fields for app actors
|
||||
if (!isUser) {
|
||||
const emailReadable = await (user.uuid === actor.user.uuid
|
||||
? emailCheck
|
||||
: canReadEmail(user.uuid));
|
||||
if (!emailReadable) {
|
||||
const canReadEmail = await services.permission
|
||||
.check(actor, `user:${user.uuid}:email:read`)
|
||||
.catch(() => false);
|
||||
if (!canReadEmail) {
|
||||
delete details.email;
|
||||
delete details.unconfirmed_email;
|
||||
}
|
||||
@@ -97,12 +292,6 @@ export const handleWhoami = async (
|
||||
details.app_name = app.uid;
|
||||
}
|
||||
|
||||
// A key core left undefined isn't built, so a listener can't fill it for an app.
|
||||
const builtKeys = isUser
|
||||
? null
|
||||
: new Set(
|
||||
Object.keys(details).filter((key) => details[key] !== undefined),
|
||||
);
|
||||
try {
|
||||
await clients.event.emitAndWait(
|
||||
'whoami.details',
|
||||
@@ -112,17 +301,9 @@ export const handleWhoami = async (
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
if (builtKeys) {
|
||||
for (const key of Object.keys(details)) {
|
||||
if (!builtKeys.has(key) && !APP_VISIBLE_LISTENER_KEYS.has(key)) {
|
||||
delete details[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const subscription = details.subscription as
|
||||
| { offering?: Record<string, unknown> }
|
||||
| undefined;
|
||||
{ offering?: Record<string, unknown> } | undefined;
|
||||
if (subscription?.offering) {
|
||||
delete subscription.offering.group;
|
||||
delete subscription.offering.benefits;
|
||||
|
||||
Generated
+174
-836
File diff suppressed because it is too large.
Load diff
+2
-3
@@ -50,7 +50,6 @@
|
||||
"build:workerLib:coverage": "cd src/puter-js && npm run build:coverage && cd ../worker && npm run build",
|
||||
"start:gui": "nodemon --exec \"node dev-server.js\" ",
|
||||
"start": "node ./tools/start.mjs",
|
||||
"start:smtp": "node ./tools/start.mjs --smtp",
|
||||
"dev": "npm start",
|
||||
"build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
|
||||
"build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build",
|
||||
@@ -77,7 +76,7 @@
|
||||
"@aws-sdk/s3-request-presigner": "^3.1028.0",
|
||||
"dedent": "^1.5.3",
|
||||
"javascript-time-ago": "^2.5.11",
|
||||
"libphonenumber-js": "1.13.15",
|
||||
"libphonenumber-js": "1.13.13",
|
||||
"miniflare": "^4.20260617.1",
|
||||
"open": "^10.1.0"
|
||||
},
|
||||
@@ -97,6 +96,6 @@
|
||||
"workerd": true
|
||||
},
|
||||
"overrides": {
|
||||
"undici": "^7.30.0"
|
||||
"undici": "^7.29.0"
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import type { PuterServer } from './server.ts';
|
||||
import { createTestUser, setupTestServer } from './testUtil.ts';
|
||||
|
||||
/**
|
||||
* Teams is optional, so core has to work when neither the service nor the store
|
||||
* is registered, which is how this test server boots.
|
||||
*/
|
||||
describe('core without a teams registration', () => {
|
||||
let server: PuterServer;
|
||||
|
||||
const makeUser = async (): Promise<{ id: number; username: string }> => {
|
||||
const username = `nt_${Math.random().toString(36).slice(2, 10)}`;
|
||||
const created = (await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
})) as unknown as { id: number };
|
||||
return { id: created.id, username };
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer({
|
||||
// Without this the allowlist check returns before asking Teams.
|
||||
teams_allowed_email_domains: ['test.local'],
|
||||
} as never);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
it('has no teams in the registry', () => {
|
||||
expect(server.services.team).toBeUndefined();
|
||||
expect(server.stores.team).toBeUndefined();
|
||||
});
|
||||
|
||||
it('refuses to boot when the config asks for teams', async () => {
|
||||
await expect(
|
||||
setupTestServer({ teams_enabled: true } as never),
|
||||
).rejects.toThrow(/teams_enabled/);
|
||||
});
|
||||
|
||||
it('deletes an account without asking what it was charged for', async () => {
|
||||
const user = await makeUser();
|
||||
await expect(
|
||||
server.services.userAccount.cascadeDelete(user.id),
|
||||
).resolves.not.toThrow();
|
||||
expect(await server.stores.user.getById(user.id)).toBeFalsy();
|
||||
});
|
||||
|
||||
it('answers the address-claim question signup asks', async () => {
|
||||
const user = await makeUser();
|
||||
const row = (await server.stores.user.getById(user.id))!;
|
||||
// Nothing is a seat, so the holder is returned as-is.
|
||||
await expect(
|
||||
server.services.signup.signupEmailHolder(row.email as string, {
|
||||
releaseSeat: true,
|
||||
}),
|
||||
).resolves.toMatchObject({ id: user.id });
|
||||
});
|
||||
|
||||
it('lists what is shared with an account, folding in no team shares', async () => {
|
||||
const user = await makeUser();
|
||||
await expect(
|
||||
server.services.share.listSharedWithMe({
|
||||
user: { id: user.id },
|
||||
} as never),
|
||||
).resolves.toMatchObject({ items: [] });
|
||||
});
|
||||
|
||||
it('refuses a share aimed at a team, rather than failing', async () => {
|
||||
// A real home directory, so the refusal is the recipient's and not
|
||||
// the entry's.
|
||||
const username = `nt_${Math.random().toString(36).slice(2, 10)}`;
|
||||
const creds = await createTestUser(server, {
|
||||
username,
|
||||
password: 'Nt-Pw-12345!',
|
||||
});
|
||||
const row = (await server.stores.user.getByUsername(username))!;
|
||||
const user = { id: row.id, username };
|
||||
const actor = { user: row } as never;
|
||||
expect(creds).toBeTruthy();
|
||||
// The recipient cannot exist, so this is a 404 and not a crash.
|
||||
await expect(
|
||||
server.services.share.share(actor, {
|
||||
recipient: { teamHandle: 'nobody' },
|
||||
mode: 'read',
|
||||
path: `/${user.username}`,
|
||||
} as never),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('serves the homepage shell with the teams tab off', async () => {
|
||||
const user = await makeUser();
|
||||
const row = (await server.stores.user.getById(user.id))!;
|
||||
let html = '';
|
||||
const res = { send: (out: string) => (html = out) };
|
||||
await server.services.homepage.send(
|
||||
{
|
||||
req: { headers: {}, query: {}, get: () => undefined },
|
||||
res,
|
||||
actor: { user: row },
|
||||
} as never,
|
||||
{ title: 'Puter' } as never,
|
||||
{} as never,
|
||||
);
|
||||
const params = JSON.parse(
|
||||
/gui\((\{.*?\})\);/s.exec(html)![1].replaceAll('\\u003c', '<'),
|
||||
);
|
||||
expect(params.teams_ui).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
ALARM_IDLE_TTL_MS,
|
||||
AlarmClient,
|
||||
MAX_TRACKED_ALARMS,
|
||||
} from './AlarmClient';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { AlarmClient } from './AlarmClient';
|
||||
import type { IConfig, IPagerConfig } from '../../types';
|
||||
import type { AlertPayload } from './types';
|
||||
|
||||
@@ -346,19 +342,6 @@ describe('AlarmClient alarm registry', () => {
|
||||
expect(client.get(shortId)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps the short id of another alarm that shares it when one is cleared', () => {
|
||||
const client = makeClient();
|
||||
// These two ids hash to the same short id.
|
||||
client.create('alarm:274', 'first');
|
||||
const shortId = client.get('alarm:274')!.shortId;
|
||||
client.create('alarm:643', 'second');
|
||||
expect(client.get('alarm:643')!.shortId).toBe(shortId);
|
||||
|
||||
client.clear('alarm:274');
|
||||
|
||||
expect(client.get(shortId)?.id).toBe('alarm:643');
|
||||
});
|
||||
|
||||
it('ignores a clear for an alarm that is not active', () => {
|
||||
const client = makeClient();
|
||||
expect(() => client.clear('never-raised')).not.toThrow();
|
||||
@@ -445,110 +428,6 @@ describe('AlarmClient alarm registry', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient alarm eviction', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('forgets alarms that stay quiet past the idle window, under both ids', () => {
|
||||
const client = makeClient();
|
||||
const ids = Array.from(
|
||||
{ length: 50 },
|
||||
(_, i) => `metering_usage_dropped:user-${i}`,
|
||||
);
|
||||
for (const id of ids) client.create(id, 'usage dropped');
|
||||
const shortIds = ids.map((id) => client.get(id)!.shortId);
|
||||
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS + 1);
|
||||
client.create('unrelated', 'the next alarm sweeps the registry');
|
||||
|
||||
expect(ids.filter((id) => client.get(id))).toEqual([]);
|
||||
expect(shortIds.filter((shortId) => client.get(shortId))).toEqual([]);
|
||||
expect(client.get('unrelated')).toBeDefined();
|
||||
});
|
||||
|
||||
it('treats a recurrence after the idle window as a new alarm', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
client.create('scan:failed', 'first');
|
||||
client.create('scan:failed', 'second');
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS + 1);
|
||||
client.create('scan:failed', 'back again');
|
||||
|
||||
expect(seen.at(-1)).toMatchObject({ repeatCount: 1, isRepeat: false });
|
||||
// A fresh start keeps per-occurrence incident keys from colliding.
|
||||
expect(seen.at(-1)!.dedupKey).not.toBe(seen[0].dedupKey);
|
||||
});
|
||||
|
||||
it('still treats a repeat inside the window as a repeat after a sweep', () => {
|
||||
const client = makeClient();
|
||||
const seen = capture(client);
|
||||
|
||||
for (let i = 0; i < 10; i++) client.create('flap', 'same fault');
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS - 1000);
|
||||
client.create('other', 'sweeps the registry');
|
||||
client.create('flap', 'same fault');
|
||||
|
||||
// Occurrence 11 is past the burst, so nothing new goes out for it.
|
||||
expect(seen.map((alert) => alert.id)).toEqual([
|
||||
...Array(10).fill('flap'),
|
||||
'other',
|
||||
]);
|
||||
expect(client.get('flap')?.count).toBe(11);
|
||||
});
|
||||
|
||||
it('keeps an alarm that keeps recurring for longer than the window', () => {
|
||||
const client = makeClient();
|
||||
|
||||
for (let i = 0; i < 6; i++) {
|
||||
client.create('hot', 'still broken');
|
||||
vi.advanceTimersByTime(ALARM_IDLE_TTL_MS / 2);
|
||||
client.create(`noise:${i}`, 'sweeps the registry');
|
||||
}
|
||||
|
||||
expect(client.get('hot')?.count).toBe(6);
|
||||
});
|
||||
|
||||
it('drops the least recently raised alarm once full', () => {
|
||||
const client = makeClient();
|
||||
|
||||
for (let i = 0; i < MAX_TRACKED_ALARMS; i++) {
|
||||
client.create(`user:${i}`, 'per-user alarm');
|
||||
vi.advanceTimersByTime(1);
|
||||
}
|
||||
// Raised again, so user:1 is now the one seen longest ago.
|
||||
client.create('user:0', 'per-user alarm');
|
||||
client.create('one-more', 'over the cap');
|
||||
|
||||
expect(client.get('user:1')).toBeUndefined();
|
||||
expect(client.get('user:0')?.count).toBe(2);
|
||||
expect(client.get('user:2')).toBeDefined();
|
||||
expect(client.get('one-more')).toBeDefined();
|
||||
});
|
||||
|
||||
it('never holds more than the cap', () => {
|
||||
const client = makeClient();
|
||||
const ids = Array.from(
|
||||
{ length: MAX_TRACKED_ALARMS * 3 },
|
||||
(_, i) => `user:${i}`,
|
||||
);
|
||||
|
||||
for (const id of ids) client.create(id, 'per-user alarm');
|
||||
|
||||
const held = ids.filter((id) => client.get(id));
|
||||
expect(held).toHaveLength(MAX_TRACKED_ALARMS);
|
||||
expect(held[0]).toBe(`user:${MAX_TRACKED_ALARMS * 2}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AlarmClient known-error rules', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
@@ -61,24 +61,14 @@ interface RegisteredHandler {
|
||||
/** Severity used when neither the call site nor config picks one. */
|
||||
const FALLBACK_SEVERITY: PagerSeverity = 'critical';
|
||||
/**
|
||||
* How many recent occurrences an alarm keeps. A hot alarm repeats for as long
|
||||
* as the fault lasts, so retaining every occurrence means retaining every
|
||||
* message and field set it was ever raised with, request bodies and actors
|
||||
* included. The last few are what a human reads; the rest is only a count, and
|
||||
* `count` keeps that.
|
||||
* How many recent occurrences an alarm keeps. An alarm is never cleared unless
|
||||
* something calls `clear`, and a hot one repeats for as long as the fault lasts
|
||||
* — so retaining every occurrence means retaining every message and field set
|
||||
* it was ever raised with, request bodies and actors included, for the life of
|
||||
* the process. The last few are what a human reads; the rest is only a count,
|
||||
* and `count` keeps that.
|
||||
*/
|
||||
const OCCURRENCE_HISTORY_LIMIT = 20;
|
||||
/**
|
||||
* An alarm that hasn't recurred for this long is forgotten; its next occurrence
|
||||
* is a new alarm with a fresh count. Kept far past the chat transport's repeat
|
||||
* throttle so a slowly recurring fault still counts as one alarm.
|
||||
*/
|
||||
export const ALARM_IDLE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
/**
|
||||
* Most alarms held at once. Ids that embed a user or key can outrun the idle
|
||||
* window; past this the least recently raised alarm is dropped.
|
||||
*/
|
||||
export const MAX_TRACKED_ALARMS = 1000;
|
||||
|
||||
// A fast-recurring fault repeats far faster than it is useful to report, and
|
||||
// every report allocates a log line and a payload. Report every occurrence up
|
||||
@@ -197,7 +187,6 @@ function cleanFields(fields: AlarmFields): Record<string, string> {
|
||||
* `pager.severityOverrides` in {@link IConfig}.
|
||||
*/
|
||||
export class AlarmClient extends PuterClient {
|
||||
/** Ordered by last occurrence: a repeat moves its alarm to the back. */
|
||||
private alarms = new Map<string, Alarm>();
|
||||
private aliases = new Map<string, Alarm>();
|
||||
private alertHandlers: RegisteredHandler[] = [];
|
||||
@@ -301,8 +290,7 @@ export class AlarmClient extends PuterClient {
|
||||
|
||||
/**
|
||||
* Create or update an alarm. If the alarm ID already exists, the occurrence
|
||||
* count is incremented and a repeat alert is dispatched. An alarm idle for
|
||||
* {@link ALARM_IDLE_TTL_MS} is forgotten, so its next occurrence is new.
|
||||
* count is incremented and a repeat alert is dispatched.
|
||||
*
|
||||
* `severity` decides where the alarm lands:
|
||||
*
|
||||
@@ -333,12 +321,9 @@ export class AlarmClient extends PuterClient {
|
||||
return;
|
||||
}
|
||||
|
||||
this.evict();
|
||||
const existing = this.alarms.get(id);
|
||||
|
||||
if (existing) {
|
||||
this.alarms.delete(id);
|
||||
this.alarms.set(id, existing);
|
||||
this.recordOccurrence(existing, message, fields);
|
||||
this.handleRepeat(existing);
|
||||
return;
|
||||
@@ -363,7 +348,6 @@ export class AlarmClient extends PuterClient {
|
||||
this.alarms.set(id, alarm);
|
||||
this.aliases.set(alarm.shortId, alarm);
|
||||
this.recordOccurrence(alarm, message, fields);
|
||||
this.evict();
|
||||
this.handleNew(alarm);
|
||||
}
|
||||
|
||||
@@ -372,7 +356,8 @@ export class AlarmClient extends PuterClient {
|
||||
const alarm = this.alarms.get(id);
|
||||
if (!alarm) return;
|
||||
|
||||
this.forget(alarm);
|
||||
this.alarms.delete(id);
|
||||
this.aliases.delete(alarm.shortId);
|
||||
console.log(`[alarm] CLEAR ${displayId(alarm)} :: ${alarm.message}`);
|
||||
}
|
||||
|
||||
@@ -410,32 +395,6 @@ export class AlarmClient extends PuterClient {
|
||||
|
||||
// -- Internals ----------------------------------------------------
|
||||
|
||||
/**
|
||||
* Drop idle alarms, then the least recently raised ones past the cap. The
|
||||
* map's order puts both at the front, so this stops at the first keeper.
|
||||
*/
|
||||
private evict(): void {
|
||||
const now = Date.now();
|
||||
for (const alarm of this.alarms.values()) {
|
||||
const lastSeen = alarm.timestamps.at(-1) ?? alarm.started;
|
||||
if (
|
||||
this.alarms.size <= MAX_TRACKED_ALARMS &&
|
||||
now - lastSeen < ALARM_IDLE_TTL_MS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
this.forget(alarm);
|
||||
}
|
||||
}
|
||||
|
||||
private forget(alarm: Alarm): void {
|
||||
this.alarms.delete(alarm.id);
|
||||
// Short ids can collide; leave an alias that now names another alarm.
|
||||
if (this.aliases.get(alarm.shortId) === alarm) {
|
||||
this.aliases.delete(alarm.shortId);
|
||||
}
|
||||
}
|
||||
|
||||
private recordOccurrence(
|
||||
alarm: Alarm,
|
||||
message: string,
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
*/
|
||||
|
||||
import type { ISlackAlertConfig, PagerSeverity } from '../../types';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap';
|
||||
import type { AlertHandler, AlertPayload } from './types';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 5000;
|
||||
@@ -124,14 +123,30 @@ export function createSlackAlertHandler(
|
||||
): AlertHandler {
|
||||
const webhookUrl = conf.webhookUrl as string;
|
||||
const throttleMs = conf.repeatThrottleMs ?? DEFAULT_REPEAT_THROTTLE_MS;
|
||||
// A zero or negative throttle expires every entry on arrival.
|
||||
const lastPosted = new BoundedTtlMap<string, true>({
|
||||
maxEntries: MAX_THROTTLE_ENTRIES,
|
||||
ttlMs: throttleMs,
|
||||
});
|
||||
const lastPosted = new Map<string, number>();
|
||||
|
||||
const shouldPost = (alert: AlertPayload): boolean => {
|
||||
if (throttleMs <= 0) return true;
|
||||
const now = Date.now();
|
||||
const previous = lastPosted.get(alert.id);
|
||||
if (previous !== undefined && now - previous < throttleMs) return false;
|
||||
|
||||
if (lastPosted.size >= MAX_THROTTLE_ENTRIES) {
|
||||
for (const [id, at] of lastPosted) {
|
||||
if (now - at >= throttleMs) lastPosted.delete(id);
|
||||
}
|
||||
// Still full of live entries — drop the oldest to stay bounded.
|
||||
if (lastPosted.size >= MAX_THROTTLE_ENTRIES) {
|
||||
const oldest = lastPosted.keys().next().value;
|
||||
if (oldest !== undefined) lastPosted.delete(oldest);
|
||||
}
|
||||
}
|
||||
lastPosted.set(alert.id, now);
|
||||
return true;
|
||||
};
|
||||
|
||||
return async (alert) => {
|
||||
if (!lastPosted.shouldEmit(alert.id)) return;
|
||||
if (!shouldPost(alert)) return;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
|
||||
@@ -45,15 +45,10 @@ class FixtureDatabaseClient extends AbstractDatabaseClient {
|
||||
constructor(
|
||||
private readonly replicaRows: Record<string, unknown>[] | Error,
|
||||
private readonly primaryRows: Record<string, unknown>[] = [],
|
||||
private readonly replica = true,
|
||||
) {
|
||||
super(config('fixture'));
|
||||
}
|
||||
|
||||
protected override hasReadReplica(): boolean {
|
||||
return this.replica;
|
||||
}
|
||||
|
||||
override async read(query: string): Promise<Record<string, unknown>[]> {
|
||||
this.reads.push(query);
|
||||
if (this.replicaRows instanceof Error) throw this.replicaRows;
|
||||
@@ -132,22 +127,14 @@ describe('AbstractDatabaseClient — replica-aware reads', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('surfaces the primary error when both reads fail', async () => {
|
||||
const client = new FixtureDatabaseClient(new Error('replica down'));
|
||||
client.pread = async () => {
|
||||
throw new Error('primary down');
|
||||
};
|
||||
await expect(client.tryHardRead('SELECT 1')).rejects.toThrow(
|
||||
'primary down',
|
||||
it('names the failing query when a required read finds nothing', async () => {
|
||||
const client = new FixtureDatabaseClient([], []);
|
||||
await expect(
|
||||
client.requireRead('SELECT * FROM `user` WHERE `id` = ?'),
|
||||
).rejects.toThrow(
|
||||
'required read returned no rows: SELECT * FROM `user` WHERE `id` = ?',
|
||||
);
|
||||
});
|
||||
|
||||
it('issues a single read when there is no replica', async () => {
|
||||
const client = new FixtureDatabaseClient([], [{ id: 2 }], false);
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([]);
|
||||
expect(client.reads).toEqual(['SELECT 1']);
|
||||
expect(client.preads).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AbstractDatabaseClient — SQL generation', () => {
|
||||
@@ -248,25 +235,6 @@ describe('AbstractDatabaseClient — SQL generation', () => {
|
||||
expect(mysql().returningIdClause()).toBe('');
|
||||
});
|
||||
|
||||
it('stamps a timestamp from unix seconds in each dialect', () => {
|
||||
expect(sqlite().timestampFromUnixSql()).toBe(
|
||||
"datetime(?, 'unixepoch')",
|
||||
);
|
||||
expect(mysql().timestampFromUnixSql()).toBe('FROM_UNIXTIME(?)');
|
||||
expect(postgres().timestampFromUnixSql()).toBe('to_timestamp(?)');
|
||||
});
|
||||
|
||||
it('describes a stored timestamp the way each driver reads it back', () => {
|
||||
const seconds = Date.UTC(2026, 9, 11, 8, 5, 9) / 1000;
|
||||
expect(sqlite().timestampValue(seconds)).toBe('2026-10-11 08:05:09');
|
||||
expect(mysql().timestampValue(seconds)).toEqual(
|
||||
new Date(seconds * 1000),
|
||||
);
|
||||
expect(postgres().timestampValue(seconds)).toEqual(
|
||||
new Date(seconds * 1000),
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to `otherwise` for an engine with no explicit choice', () => {
|
||||
const client = new FixtureDatabaseClient([]);
|
||||
expect(
|
||||
|
||||
@@ -17,23 +17,13 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { isAbsolute, resolve as resolvePath } from 'path';
|
||||
import type { IConfig } from '../../types';
|
||||
import { Span } from '../../util/span.js';
|
||||
import { PuterClient } from '../types';
|
||||
import { compareMigrationFilenames } from './migrationFilenames.js';
|
||||
|
||||
export interface WriteResult {
|
||||
insertId: number | bigint;
|
||||
affectedRows: number;
|
||||
anyRowsAffected: boolean;
|
||||
/**
|
||||
* Warnings the statement raised; nonzero means a value may have been stored
|
||||
* differently from how it was sent. Absent when the engine can't vouch that
|
||||
* a clean insert stored exactly what was sent, on a fresh id.
|
||||
*/
|
||||
warningStatus?: number;
|
||||
}
|
||||
|
||||
export interface BatchEntry {
|
||||
@@ -43,12 +33,6 @@ export interface BatchEntry {
|
||||
|
||||
type SqlJsonPath = readonly [string, ...string[]];
|
||||
|
||||
/** The slice of a pooled connection that migrations need. */
|
||||
export interface MigrationConnection {
|
||||
query(sql: string): Promise<unknown>;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base database client. Subclasses must override every method that throws here.
|
||||
*
|
||||
@@ -120,133 +104,39 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
return this.write(sql, values);
|
||||
}
|
||||
|
||||
/** Whether `read()` goes to a separate replica rather than the primary. */
|
||||
protected hasReadReplica(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* The replica leg of `tryHardRead`. Override when `read()` falls back to
|
||||
* the primary by itself, or a replica failure reads the primary twice.
|
||||
* Like `read()` but falls back to the primary when read-replicas are in
|
||||
* use. Subclasses may override with replica-aware logic; the default
|
||||
* delegates to `pread()`.
|
||||
*/
|
||||
protected replicaRead(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Like `read()`, but an empty or failed replica read falls back to the
|
||||
* primary, so replication lag can't hide a row that was just written. The
|
||||
* primary read starts alongside the replica one rather than after it.
|
||||
* Without a replica this is a single `read()`.
|
||||
*/
|
||||
@Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
|
||||
async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (!this.hasReadReplica()) return this.read(query, params);
|
||||
const primary = this.pread(query, params);
|
||||
primary.catch(() => {});
|
||||
|
||||
const primary = settle(this.pread(query, params));
|
||||
try {
|
||||
const rows = await this.replicaRead(query, params);
|
||||
if (rows.length > 0) return rows;
|
||||
const rows = await this.read(query, params);
|
||||
if (rows.length > 0) {
|
||||
return rows;
|
||||
}
|
||||
} catch {
|
||||
// replica failed — fall through to primary
|
||||
}
|
||||
const result = await primary;
|
||||
if ('error' in result) throw result.error;
|
||||
return result.value;
|
||||
return primary;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the `.sql` files named `<engineName>*` in each configured migration
|
||||
* directory, in filename order. Files must be idempotent: nothing records
|
||||
* which ones already ran. A failure aborts startup. `transactional` wraps
|
||||
* each file in BEGIN/COMMIT, for engines whose DDL can roll back.
|
||||
*/
|
||||
protected async applyMigrationPaths(
|
||||
connect: () => Promise<MigrationConnection>,
|
||||
splitStatements: (contents: string) => string[],
|
||||
{ transactional }: { transactional: boolean },
|
||||
): Promise<void> {
|
||||
const paths = this.config.database?.migrationPaths;
|
||||
if (!paths || paths.length === 0) return;
|
||||
|
||||
const tag = `[${this.engineName}]`;
|
||||
const conn = await connect();
|
||||
try {
|
||||
for (const rawPath of paths) {
|
||||
const dir = isAbsolute(rawPath)
|
||||
? rawPath
|
||||
: resolvePath(process.cwd(), rawPath);
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = readdirSync(dir)
|
||||
.filter(
|
||||
(f) =>
|
||||
f.endsWith('.sql') &&
|
||||
f.startsWith(this.engineName),
|
||||
)
|
||||
.sort(compareMigrationFilenames);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`${tag} migration path is unreadable: ${dir}`,
|
||||
{
|
||||
cause: e,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log(`${tag} no migrations in ${dir}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${tag} running migrations from ${dir}: ${files.length} file(s)`,
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
const contents = readFileSync(
|
||||
resolvePath(dir, file),
|
||||
'utf8',
|
||||
);
|
||||
const statements = splitStatements(contents);
|
||||
if (transactional) await conn.query('BEGIN');
|
||||
try {
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
try {
|
||||
await conn.query(statements[i]);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`${tag} failed to apply ${file} at statement ${i}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
}
|
||||
if (transactional) await conn.query('COMMIT');
|
||||
} catch (e) {
|
||||
if (transactional) {
|
||||
try {
|
||||
await conn.query('ROLLBACK');
|
||||
} catch {
|
||||
// the original failure is the one worth surfacing
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
console.log(
|
||||
`${tag} applied ${file} (${statements.length} statements)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
/** Like `tryHardRead()` but throws when the result set is empty. */
|
||||
async requireRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const rows = await this.tryHardRead(query, params);
|
||||
if (rows.length === 0) {
|
||||
throw new Error(`required read returned no rows: ${query}`);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -336,30 +226,6 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Whole `periodSeconds` buckets since the epoch, from a unix-seconds
|
||||
* column. MySQL's `/` yields a decimal, which would split a bucket.
|
||||
*/
|
||||
periodBucket(secondsExpression: string, periodSeconds: number): string {
|
||||
return this.case({
|
||||
mysql: `(${secondsExpression} DIV ${periodSeconds})`,
|
||||
otherwise: `(${secondsExpression} / ${periodSeconds})`,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts each (`userExpression`, bucket) pair once. `COUNT(DISTINCT a, b)`
|
||||
* is MySQL-only, so the pair folds into one number — injective while the
|
||||
* bucket stays under the multiplier, which hourly reaches in the 2980s.
|
||||
*/
|
||||
countDistinctUserPeriods(
|
||||
userExpression: string,
|
||||
secondsExpression: string,
|
||||
periodSeconds: number,
|
||||
): string {
|
||||
return `COUNT(DISTINCT ${userExpression} * 10000000 + ${this.periodBucket(secondsExpression, periodSeconds)})`;
|
||||
}
|
||||
|
||||
nullCoalesce(...expressions: readonly string[]): string {
|
||||
if (expressions.length === 0) {
|
||||
throw new Error('nullCoalesce requires at least one expression');
|
||||
@@ -374,43 +240,7 @@ export class AbstractDatabaseClient extends PuterClient {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A timestamp column value from one bound parameter in unix seconds, stored
|
||||
* the way `CURRENT_TIMESTAMP` would store that instant.
|
||||
*/
|
||||
timestampFromUnixSql(): string {
|
||||
return this.case({
|
||||
sqlite: "datetime(?, 'unixepoch')",
|
||||
postgres: 'to_timestamp(?)',
|
||||
otherwise: 'FROM_UNIXTIME(?)',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* What a read of a timestamp column holding `seconds` returns: sqlite gives
|
||||
* back the UTC text it stored, mysql2 and pg a `Date` read in node's time
|
||||
* zone (so this matches when node and the database session share one).
|
||||
*/
|
||||
timestampValue(seconds: number): Date | string {
|
||||
const date = new Date(seconds * 1000);
|
||||
return this.case<Date | string>({
|
||||
sqlite: date.toISOString().slice(0, 19).replace('T', ' '),
|
||||
otherwise: date,
|
||||
});
|
||||
}
|
||||
|
||||
protected sqlStringLiteral(value: string): string {
|
||||
return `'${value.replaceAll("'", "''")}'`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Await later without an unhandled rejection in the meantime. */
|
||||
const settle = async <T>(
|
||||
promise: Promise<T>,
|
||||
): Promise<{ value: T } | { error: unknown }> => {
|
||||
try {
|
||||
return { value: await promise };
|
||||
} catch (error) {
|
||||
return { error };
|
||||
}
|
||||
};
|
||||
@@ -184,8 +184,7 @@ describe('compareMigrationFilenames', () => {
|
||||
//
|
||||
// `read()` normally goes to the replica batcher; when the replica side is
|
||||
// degraded (batcher load-shed or a transient connection error) and a real
|
||||
// replica is configured, the read retries once on the primary read batcher.
|
||||
// The write batcher is left unset, so a failover that reached it would throw.
|
||||
// replica is configured, the read retries once on the primary batcher.
|
||||
|
||||
type Batcher = { execute: ReturnType<typeof vi.fn> };
|
||||
|
||||
@@ -201,7 +200,7 @@ const makeClient = (opts: {
|
||||
// inject the batchers directly. Configuration enum: SINGLE=0, REPLICA=1.
|
||||
Object.assign(client as unknown as Record<string, unknown>, {
|
||||
dbReplica: opts.replica,
|
||||
dbPrimaryRead: opts.primary,
|
||||
db: opts.primary,
|
||||
configuration: opts.multiNode === false ? 0 : 1,
|
||||
});
|
||||
return client;
|
||||
@@ -285,10 +284,6 @@ const startClient = async (
|
||||
// must answer with one row-set per statement plus one for the SELECT 1.
|
||||
const rowSets = (...sets: unknown[][]) => [[...sets, [{ 1: 1 }]], undefined];
|
||||
|
||||
// Writes go out one statement at a time, so the driver answers with just the
|
||||
// result header.
|
||||
const writeHeader = (header: Record<string, unknown>) => [header, undefined];
|
||||
|
||||
beforeEach(() => {
|
||||
createdPools.length = 0;
|
||||
createPoolMock.mockClear();
|
||||
@@ -308,9 +303,6 @@ describe('MySQLDatabaseClient — pool construction', () => {
|
||||
expect(createdPools[0].poolConfig).toEqual({
|
||||
maxPreparedStatements: 900,
|
||||
connectionLimit: 30,
|
||||
maxIdle: 29,
|
||||
idleTimeout: 60_000,
|
||||
gracefulEnd: true,
|
||||
enableKeepAlive: true,
|
||||
host: 'db.internal',
|
||||
port: 3307,
|
||||
@@ -321,32 +313,6 @@ describe('MySQLDatabaseClient — pool construction', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// mysql2 skips its idle reaper unless maxIdle < connectionLimit, and an
|
||||
// unreaped connection is closed by the server under the pool's feet.
|
||||
it('keeps the idle reaper on when a pool sets its own connection limit', async () => {
|
||||
await startClient({
|
||||
replica: { host: 'replica.internal', connectionLimit: 10 },
|
||||
});
|
||||
|
||||
expect(createdPools[1].poolConfig).toMatchObject({
|
||||
connectionLimit: 10,
|
||||
maxIdle: 9,
|
||||
idleTimeout: 60_000,
|
||||
});
|
||||
});
|
||||
|
||||
// mysql2 reads 0 as unlimited; a maxIdle of -1 would crash its reaper.
|
||||
it('leaves maxIdle to mysql2 for an unlimited pool', async () => {
|
||||
await startClient({
|
||||
replica: { host: 'replica.internal', connectionLimit: 0 },
|
||||
});
|
||||
|
||||
expect(createdPools[1].poolConfig).toMatchObject({
|
||||
connectionLimit: 0,
|
||||
});
|
||||
expect(createdPools[1].poolConfig).not.toHaveProperty('maxIdle');
|
||||
});
|
||||
|
||||
it('falls back to loopback defaults when the endpoint is unspecified', async () => {
|
||||
await startClient();
|
||||
|
||||
@@ -435,129 +401,10 @@ describe('MySQLDatabaseClient — query interface', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ method: 'pread', poolIndex: 0, maxTimeInQueue: 30 },
|
||||
{ method: 'read', poolIndex: 1, maxTimeInQueue: 10 },
|
||||
] as const)(
|
||||
'$method: flushes lone reads at once until 8 are in flight, then waits $maxTimeInQueue ms and coalesces',
|
||||
async ({ method, poolIndex, maxTimeInQueue }) => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const pool = createdPools[poolIndex];
|
||||
const held: Array<() => void> = [];
|
||||
pool.respond = (sql) =>
|
||||
new Promise((resolve) => {
|
||||
const sets = sql
|
||||
.split(';')
|
||||
.slice(0, -1)
|
||||
.map((stmt) => [{ stmt }]);
|
||||
held.push(() => resolve(rowSets(...sets)));
|
||||
});
|
||||
vi.useFakeTimers({
|
||||
toFake: [
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
try {
|
||||
const reads: Promise<unknown>[] = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
reads.push(client[method](`SELECT ${i}`));
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
}
|
||||
expect(pool.calls).toHaveLength(8);
|
||||
|
||||
reads.push(client[method]('SELECT 8'));
|
||||
await vi.advanceTimersByTimeAsync(maxTimeInQueue - 1);
|
||||
reads.push(client[method]('SELECT 9'));
|
||||
expect(pool.calls).toHaveLength(8);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(pool.calls.map((c) => c.sql).at(-1)).toBe(
|
||||
'SELECT 8;SELECT 9; SELECT 1',
|
||||
);
|
||||
|
||||
for (const release of held.splice(0)) release();
|
||||
const rows = await Promise.all(reads);
|
||||
expect(rows.at(-2)).toEqual([{ stmt: 'SELECT 8' }]);
|
||||
expect(rows.at(-1)).toEqual([{ stmt: 'SELECT 9' }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ setup: 'single node', replica: undefined, waits: true },
|
||||
{ setup: 'replica', replica: { host: 'replica' }, waits: false },
|
||||
])(
|
||||
'$setup: 8 primary reads in flight hold back a lone read: $waits',
|
||||
async ({ replica, waits }) => {
|
||||
const client = await startClient(replica ? { replica } : {});
|
||||
const held: Array<() => void> = [];
|
||||
for (const pool of createdPools) {
|
||||
pool.respond = (sql) =>
|
||||
new Promise((resolve) => {
|
||||
const sets = sql
|
||||
.split(';')
|
||||
.slice(0, -1)
|
||||
.map((stmt) => [{ stmt }]);
|
||||
held.push(() => resolve(rowSets(...sets)));
|
||||
});
|
||||
}
|
||||
const sent = () => createdPools.flatMap((pool) => pool.calls);
|
||||
vi.useFakeTimers({
|
||||
toFake: [
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
try {
|
||||
const reads: Promise<unknown>[] = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
reads.push(client.pread(`SELECT ${i}`));
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
}
|
||||
reads.push(client.read('SELECT 8'));
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toHaveLength(waits ? 8 : 9);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
expect(sent()).toHaveLength(9);
|
||||
|
||||
for (const release of held.splice(0)) release();
|
||||
await Promise.all(reads);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('sends a write on its own, outside a transaction', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => writeHeader({ affectedRows: 1 });
|
||||
|
||||
await client.write('UPDATE `user` SET `username` = ? WHERE `id` = ?', [
|
||||
'ada',
|
||||
1,
|
||||
]);
|
||||
|
||||
expect(pool.calls).toEqual([
|
||||
{
|
||||
sql: 'UPDATE `user` SET `username` = ? WHERE `id` = ?',
|
||||
values: ['ada', 1],
|
||||
},
|
||||
]);
|
||||
expect(pool.connections[0].beginTransaction).not.toHaveBeenCalled();
|
||||
expect(pool.connections[0].commit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('maps the mysql result header onto the shared write result', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () =>
|
||||
writeHeader({ insertId: 42, affectedRows: 2 });
|
||||
rowSets({ insertId: 42, affectedRows: 2 } as unknown as unknown[]);
|
||||
|
||||
await expect(
|
||||
client.write('UPDATE `user` SET `username` = ? WHERE `id` = ?', [
|
||||
@@ -571,26 +418,9 @@ describe('MySQLDatabaseClient — query interface', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('passes the warning count through, so callers can tell a clean write', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () =>
|
||||
writeHeader({ insertId: 9, affectedRows: 1, warningStatus: 1 });
|
||||
|
||||
await expect(
|
||||
client.write('INSERT IGNORE INTO `user` (`username`) VALUES (?)', [
|
||||
'ada',
|
||||
]),
|
||||
).resolves.toEqual({
|
||||
insertId: 9,
|
||||
affectedRows: 1,
|
||||
anyRowsAffected: true,
|
||||
warningStatus: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('reports no rows affected when the header omits the counters', async () => {
|
||||
const client = await startClient();
|
||||
createdPools[0].respond = () => writeHeader({});
|
||||
createdPools[0].respond = () => rowSets({} as unknown as unknown[]);
|
||||
|
||||
await expect(client.write('DELETE FROM `user`')).resolves.toEqual({
|
||||
insertId: 0,
|
||||
@@ -602,14 +432,17 @@ describe('MySQLDatabaseClient — query interface', () => {
|
||||
it('builds an INSERT with quoted identifiers and positional params', async () => {
|
||||
const client = await startClient();
|
||||
const pool = createdPools[0];
|
||||
pool.respond = () => writeHeader({ insertId: 7, affectedRows: 1 });
|
||||
pool.respond = () =>
|
||||
rowSets({ insertId: 7, affectedRows: 1 } as unknown as unknown[]);
|
||||
|
||||
await expect(
|
||||
client.insert('user', { username: 'ada', email: null }),
|
||||
).resolves.toMatchObject({ insertId: 7 });
|
||||
|
||||
expect(pool.calls[0]).toEqual({
|
||||
sql: 'INSERT INTO `user` (`username`, `email`) VALUES (?, ?)',
|
||||
sql:
|
||||
'INSERT INTO `user` (`username`, `email`) VALUES (?, ?); ' +
|
||||
'SELECT 1',
|
||||
values: ['ada', null],
|
||||
});
|
||||
});
|
||||
@@ -735,21 +568,6 @@ describe('MySQLDatabaseClient — tryHardRead', () => {
|
||||
{ from: 'primary' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('reads the primary once when the replica fails in a way read() fails over on', async () => {
|
||||
const replica = {
|
||||
execute: vi.fn().mockRejectedValue(codedError('dbBatchFailed')),
|
||||
};
|
||||
const primary = {
|
||||
execute: vi.fn().mockResolvedValue([[{ from: 'primary' }]]),
|
||||
};
|
||||
const client = makeClient({ replica, primary });
|
||||
|
||||
await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'primary' },
|
||||
]);
|
||||
expect(primary.execute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — migrations', () => {
|
||||
@@ -827,6 +645,46 @@ describe('MySQLDatabaseClient — migrations', () => {
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient — pool lifecycle', () => {
|
||||
it('replaces and closes the previous primary pool on reinit', async () => {
|
||||
const client = await startClient();
|
||||
const original = createdPools[0];
|
||||
|
||||
client.reinitPrimary();
|
||||
|
||||
expect(createdPools).toHaveLength(2);
|
||||
expect(original.ended).toBe(true);
|
||||
// Single-node: the replica batcher must follow the new primary.
|
||||
createdPools[1].respond = () => rowSets([{ from: 'new-primary' }]);
|
||||
await expect(client.read('SELECT 1')).resolves.toEqual([
|
||||
{ from: 'new-primary' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('replaces only the replica pool on replica reinit', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
const [primary, originalReplica] = createdPools;
|
||||
|
||||
client.reinitReplica();
|
||||
|
||||
expect(createdPools).toHaveLength(3);
|
||||
expect(originalReplica.ended).toBe(true);
|
||||
expect(primary.ended).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores a replica reinit when no replica is configured', async () => {
|
||||
const client = await startClient();
|
||||
client.reinitReplica();
|
||||
expect(createdPools).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('ignores reinit once shutdown has started', async () => {
|
||||
const client = await startClient();
|
||||
await client.onServerPrepareShutdown();
|
||||
client.reinitPrimary();
|
||||
expect(createdPools).toHaveLength(1);
|
||||
await client.onServerShutdown();
|
||||
});
|
||||
|
||||
it('closes both pools on shutdown when a replica is configured', async () => {
|
||||
const client = await startClient({ replica: { host: 'replica' } });
|
||||
await client.onServerShutdown();
|
||||
@@ -870,3 +728,108 @@ describe('MySQLDatabaseClient — pool lifecycle', () => {
|
||||
await expect(client.onServerShutdown()).rejects.toThrow('end exploded');
|
||||
});
|
||||
});
|
||||
|
||||
describe('MySQLDatabaseClient.readWithRetry', () => {
|
||||
const codedFailure = (code: string) => {
|
||||
const err = new Error(code) as Error & { code: string };
|
||||
err.code = code;
|
||||
return err;
|
||||
};
|
||||
|
||||
it('classifies transient connection failures as retriable', () => {
|
||||
expect(
|
||||
MySQLDatabaseClient.isRetriableError(codedFailure('ECONNRESET')),
|
||||
).toBe(true);
|
||||
expect(
|
||||
MySQLDatabaseClient.isRetriableError(codedFailure('ER_DUP_ENTRY')),
|
||||
).toBe(false);
|
||||
expect(
|
||||
MySQLDatabaseClient.isRetriableError(new Error('Connection lost')),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('retries a transient failure and returns the eventual result', async () => {
|
||||
const client = await startClient();
|
||||
let attempts = 0;
|
||||
const operation = vi.fn(async () => {
|
||||
attempts += 1;
|
||||
if (attempts < 3) throw codedFailure('PROTOCOL_CONNECTION_LOST');
|
||||
return [{ ok: 1 }];
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.readWithRetry('health', operation, { baseBackoffMs: 0 }),
|
||||
).resolves.toEqual([{ ok: 1 }]);
|
||||
expect(operation).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('gives up after the attempt budget is spent', async () => {
|
||||
const client = await startClient();
|
||||
const operation = vi.fn(async () => {
|
||||
throw codedFailure('ETIMEDOUT');
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.readWithRetry('health', operation, {
|
||||
maxAttempts: 2,
|
||||
baseBackoffMs: 0,
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'ETIMEDOUT' });
|
||||
expect(operation).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('never retries a deterministic SQL error', async () => {
|
||||
const client = await startClient();
|
||||
const operation = vi.fn(async () => {
|
||||
throw codedFailure('ER_PARSE_ERROR');
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.readWithRetry('health', operation, { baseBackoffMs: 0 }),
|
||||
).rejects.toMatchObject({ code: 'ER_PARSE_ERROR' });
|
||||
expect(operation).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('stops retrying once shutdown has started', async () => {
|
||||
const client = await startClient();
|
||||
await client.onServerPrepareShutdown();
|
||||
const operation = vi.fn(async () => {
|
||||
throw codedFailure('ECONNRESET');
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.readWithRetry('health', operation, { baseBackoffMs: 0 }),
|
||||
).rejects.toMatchObject({ code: 'ECONNRESET' });
|
||||
expect(operation).toHaveBeenCalledTimes(1);
|
||||
await client.onServerShutdown();
|
||||
});
|
||||
|
||||
it('applies jittered exponential backoff capped at maxBackoffMs', async () => {
|
||||
const client = await startClient();
|
||||
const delays: number[] = [];
|
||||
const realSetTimeout = globalThis.setTimeout;
|
||||
const timeoutSpy = vi
|
||||
.spyOn(globalThis, 'setTimeout')
|
||||
.mockImplementation(((fn: () => void, ms?: number) => {
|
||||
delays.push(ms ?? 0);
|
||||
return realSetTimeout(fn, 0);
|
||||
}) as typeof setTimeout);
|
||||
|
||||
try {
|
||||
const operation = vi.fn(async () => {
|
||||
throw codedFailure('ECONNRESET');
|
||||
});
|
||||
await expect(
|
||||
client.readWithRetry('health', operation, {
|
||||
maxAttempts: 4,
|
||||
baseBackoffMs: 100,
|
||||
maxBackoffMs: 150,
|
||||
jitterRatio: 0,
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'ECONNRESET' });
|
||||
expect(delays).toEqual([100, 150, 150]);
|
||||
} finally {
|
||||
timeoutSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,8 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { isAbsolute, resolve as resolvePath } from 'path';
|
||||
import { metrics } from '@opentelemetry/api';
|
||||
import { createPool, ExecuteValues, type Pool } from 'mysql2';
|
||||
import { Span } from '../../util/span.js';
|
||||
@@ -24,11 +26,10 @@ import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
|
||||
import { SQLBatcher } from './SQLBatcher.js';
|
||||
import { isRetriableError } from './retriableErrors.js';
|
||||
import { splitMysqlStatements } from './splitMysqlStatements.js';
|
||||
import { compareMigrationFilenames } from './migrationFilenames.js';
|
||||
import type { IConfig } from '../../types';
|
||||
|
||||
const DEFAULT_SELECT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_CONNECTION_LIMIT = 30;
|
||||
const IDLE_TIMEOUT_MS = 60_000;
|
||||
|
||||
const replicaFailoverCounter = metrics
|
||||
.getMeter('puter-backend')
|
||||
@@ -37,7 +38,7 @@ const replicaFailoverCounter = metrics
|
||||
'Reads that failed on the replica batcher and were retried on the primary',
|
||||
});
|
||||
|
||||
export { compareMigrationFilenames } from './migrationFilenames.js';
|
||||
export { compareMigrationFilenames };
|
||||
|
||||
type PoolConfig = Parameters<typeof createPool>[0];
|
||||
|
||||
@@ -53,7 +54,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
private replicaPool!: Pool;
|
||||
private db!: SQLBatcher;
|
||||
private dbReplica!: SQLBatcher;
|
||||
/** Primary pool, SELECT-only: same rows as `db`, coalesced like the replica. */
|
||||
/** Primary pool, SELECT-only: same rows as `db`, without the transaction. */
|
||||
private dbPrimaryRead!: SQLBatcher;
|
||||
private configuration = Configuration.SINGLE;
|
||||
private shutdownStarted = false;
|
||||
@@ -78,13 +79,8 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
});
|
||||
console.log('[mysql] connected to primary');
|
||||
|
||||
// Read batchers on one pool share the cap on immediate flushes.
|
||||
const primaryReadFlushes = { count: 0 };
|
||||
this.db = this.createPrimaryBatcher(this.primaryPool);
|
||||
this.dbPrimaryRead = this.createPrimaryReadBatcher(
|
||||
this.primaryPool,
|
||||
primaryReadFlushes,
|
||||
);
|
||||
this.dbPrimaryRead = this.createPrimaryReadBatcher(this.primaryPool);
|
||||
|
||||
if (dbConf.replica) {
|
||||
this.replicaPool = this.createPool(dbConf.replica);
|
||||
@@ -95,26 +91,17 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
this.configuration = Configuration.SINGLE;
|
||||
}
|
||||
|
||||
this.dbReplica = this.createReplicaBatcher(
|
||||
this.replicaPool,
|
||||
this.configuration === Configuration.SINGLE
|
||||
? primaryReadFlushes
|
||||
: { count: 0 },
|
||||
);
|
||||
this.dbReplica = this.createReplicaBatcher(this.replicaPool);
|
||||
|
||||
await this.applyMigrationPaths(
|
||||
() => this.primaryPool.promise().getConnection(),
|
||||
splitMysqlStatements,
|
||||
{ transactional: false },
|
||||
);
|
||||
await this.runMigrations();
|
||||
}
|
||||
|
||||
override async onServerPrepareShutdown(): Promise<void> {
|
||||
if (this.shutdownStarted) return;
|
||||
this.shutdownStarted = true;
|
||||
|
||||
// Pools stay open; onServerShutdown closes them after the layers
|
||||
// above have drained.
|
||||
// Blocks reinitPrimary/reinitReplica from here on. Pools stay open;
|
||||
// onServerShutdown closes them after the layers above have drained.
|
||||
console.log('[mysql] entering drain mode');
|
||||
}
|
||||
|
||||
@@ -149,7 +136,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
throw error;
|
||||
}
|
||||
replicaFailoverCounter.add(1);
|
||||
result = await this.dbPrimaryRead.execute(query, params);
|
||||
result = await this.db.execute(query, params);
|
||||
}
|
||||
if (!result) return [];
|
||||
return (result[0] as Record<string, unknown>[]) ?? [];
|
||||
@@ -174,14 +161,12 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
const header = result[0] as {
|
||||
insertId?: number;
|
||||
affectedRows?: number;
|
||||
warningStatus?: number;
|
||||
};
|
||||
const affectedRows = header.affectedRows ?? 0;
|
||||
return {
|
||||
insertId: header.insertId ?? 0,
|
||||
affectedRows,
|
||||
anyRowsAffected: affectedRows > 0,
|
||||
warningStatus: header.warningStatus,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -213,18 +198,101 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
}
|
||||
}
|
||||
|
||||
protected override hasReadReplica(): boolean {
|
||||
return this.configuration === Configuration.REPLICA;
|
||||
@Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
|
||||
override async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (this.configuration === Configuration.SINGLE) {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
// Run both reads in parallel — prefer replica when it returns rows,
|
||||
// otherwise fall back to primary to handle replication lag.
|
||||
const primaryPromise = this.dbPrimaryRead.execute(query, params);
|
||||
try {
|
||||
const replicaResult = await this.dbReplica.execute(query, params);
|
||||
if (
|
||||
Array.isArray(replicaResult?.[0]) &&
|
||||
(replicaResult[0] as unknown[]).length > 0
|
||||
) {
|
||||
primaryPromise.catch(() => {}); // suppress unhandled rejection
|
||||
return replicaResult[0] as Record<string, unknown>[];
|
||||
}
|
||||
} catch {
|
||||
// fall through to primary
|
||||
}
|
||||
|
||||
const primaryResult = await primaryPromise;
|
||||
return (primaryResult?.[0] as Record<string, unknown>[]) ?? [];
|
||||
}
|
||||
|
||||
// No failover: `tryHardRead` already has a primary read in flight.
|
||||
protected override async replicaRead(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const result = await this.dbReplica.execute(query, params);
|
||||
if (!result) return [];
|
||||
return (result[0] as Record<string, unknown>[]) ?? [];
|
||||
// ------------------------------------------------------------------
|
||||
// Migrations
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Apply `.sql` files from each configured migration directory in order.
|
||||
* Files within a directory are sorted lexically. Files MUST be idempotent —
|
||||
* there is no per-file applied-state tracking. Failures abort startup so
|
||||
* operators see schema problems loud.
|
||||
*/
|
||||
private async runMigrations(): Promise<void> {
|
||||
const paths = this.config.database?.migrationPaths;
|
||||
if (!paths || paths.length === 0) return;
|
||||
|
||||
const conn = await this.primaryPool.promise().getConnection();
|
||||
try {
|
||||
for (const rawPath of paths) {
|
||||
const dir = isAbsolute(rawPath)
|
||||
? rawPath
|
||||
: resolvePath(process.cwd(), rawPath);
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = readdirSync(dir)
|
||||
.filter(
|
||||
(f) => f.endsWith('.sql') && f.startsWith('mysql'),
|
||||
)
|
||||
.sort(compareMigrationFilenames);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`[mysql] migration path is unreadable: ${dir}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log(`[mysql] no migrations in ${dir}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[mysql] running migrations from ${dir}: ${files.length} file(s)`,
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = resolvePath(dir, file);
|
||||
const contents = readFileSync(filePath, 'utf8');
|
||||
const statements = splitMysqlStatements(contents);
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
try {
|
||||
await conn.query(statements[i]);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`[mysql] failed to apply ${file} at statement ${i}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`[mysql] applied ${file} (${statements.length} statements)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
@@ -232,19 +300,9 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private createPool(poolConfig: PoolConfig): Pool {
|
||||
const connectionLimit =
|
||||
poolConfig.connectionLimit ?? DEFAULT_CONNECTION_LIMIT;
|
||||
const pool = createPool({
|
||||
maxPreparedStatements: 900,
|
||||
connectionLimit,
|
||||
// mysql2 only reaps idle connections when maxIdle < connectionLimit.
|
||||
// Unreaped, they sit until the server's wait_timeout closes them,
|
||||
// and the next statement sent on one fails. 0 means unlimited,
|
||||
// and a negative maxIdle would crash the reaper.
|
||||
...(connectionLimit > 1 && { maxIdle: connectionLimit - 1 }),
|
||||
idleTimeout: IDLE_TIMEOUT_MS,
|
||||
// Reaped connections say goodbye instead of dropping the socket.
|
||||
gracefulEnd: true,
|
||||
connectionLimit: 30,
|
||||
enableKeepAlive: true,
|
||||
...poolConfig,
|
||||
multipleStatements: true,
|
||||
@@ -272,28 +330,24 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
return pool;
|
||||
}
|
||||
|
||||
/**
|
||||
* One write per flush. Coalesced writes share a transaction, so unrelated
|
||||
* callers hold each other's row locks until COMMIT and deadlock; a lone
|
||||
* statement needs no BEGIN/COMMIT round trips.
|
||||
*/
|
||||
private createPrimaryBatcher(pool: Pool): SQLBatcher {
|
||||
return new SQLBatcher(pool, {
|
||||
maxBatchSize: 1,
|
||||
maxTimeInQueue: 30,
|
||||
maxBatchSize: 5,
|
||||
poolLabel: 'primary',
|
||||
acquireTimeoutMs: this.config.database?.acquireTimeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
/** Reads that must see the primary, still coalesced. */
|
||||
private createPrimaryReadBatcher(
|
||||
pool: Pool,
|
||||
inFlight: { count: number },
|
||||
): SQLBatcher {
|
||||
/**
|
||||
* Reads that must see the primary still only read, so they skip the
|
||||
* batcher's transaction wrapper. Worth a separate batcher because BEGIN and
|
||||
* COMMIT are round trips: against a primary in another region they cost
|
||||
* more than the query.
|
||||
*/
|
||||
private createPrimaryReadBatcher(pool: Pool): SQLBatcher {
|
||||
return new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 8,
|
||||
inFlight,
|
||||
maxBatchSize: 5,
|
||||
poolLabel: 'primary',
|
||||
readOnly: true,
|
||||
@@ -301,14 +355,9 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
});
|
||||
}
|
||||
|
||||
private createReplicaBatcher(
|
||||
pool: Pool,
|
||||
inFlight: { count: number },
|
||||
): SQLBatcher {
|
||||
private createReplicaBatcher(pool: Pool): SQLBatcher {
|
||||
return new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 10,
|
||||
maxInFlightImmediate: 8,
|
||||
inFlight,
|
||||
maxBatchSize: 5,
|
||||
poolLabel: 'replica',
|
||||
readOnly: true,
|
||||
@@ -316,6 +365,57 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
});
|
||||
}
|
||||
|
||||
/** Reinitialize the primary pool (e.g. after a health-check failure). */
|
||||
reinitPrimary(): void {
|
||||
if (this.shutdownStarted) return;
|
||||
|
||||
const dbConf = this.config.database!;
|
||||
const previous = this.primaryPool;
|
||||
this.primaryPool = this.createPool({
|
||||
host: dbConf.host ?? '127.0.0.1',
|
||||
port: dbConf.port ?? 3306,
|
||||
user: dbConf.user ?? 'root',
|
||||
password: dbConf.password ?? '',
|
||||
database: dbConf.database ?? 'puter',
|
||||
});
|
||||
this.db = this.createPrimaryBatcher(this.primaryPool);
|
||||
this.dbPrimaryRead = this.createPrimaryReadBatcher(this.primaryPool);
|
||||
|
||||
if (this.configuration === Configuration.SINGLE) {
|
||||
this.replicaPool = this.primaryPool;
|
||||
this.dbReplica = this.createReplicaBatcher(this.primaryPool);
|
||||
}
|
||||
|
||||
if (previous && previous !== this.primaryPool) {
|
||||
this.closePool(previous, 'reinit:primary').catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/** Reinitialize the replica pool. */
|
||||
reinitReplica(): void {
|
||||
if (this.shutdownStarted || !this.config.database?.replica) return;
|
||||
|
||||
const previous = this.replicaPool;
|
||||
this.replicaPool = this.createPool(this.config.database.replica);
|
||||
this.dbReplica = this.createReplicaBatcher(this.replicaPool);
|
||||
|
||||
if (
|
||||
previous &&
|
||||
previous !== this.replicaPool &&
|
||||
previous !== this.primaryPool
|
||||
) {
|
||||
this.closePool(previous, 'reinit:replica').catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Retry helpers (for health checks or resilient reads)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
static isRetriableError(error: unknown): boolean {
|
||||
return isRetriableError(error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replica failures worth retrying on the primary: batcher load-shed or
|
||||
* transient connection errors — never deterministic SQL errors.
|
||||
@@ -325,6 +425,52 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
|
||||
return code === 'dbBatchFailed' || isRetriableError(error);
|
||||
}
|
||||
|
||||
async readWithRetry(
|
||||
label: string,
|
||||
operation: () => Promise<unknown[]>,
|
||||
opts?: {
|
||||
maxAttempts?: number;
|
||||
baseBackoffMs?: number;
|
||||
maxBackoffMs?: number;
|
||||
jitterRatio?: number;
|
||||
},
|
||||
): Promise<unknown[]> {
|
||||
const maxAttempts = opts?.maxAttempts ?? 3;
|
||||
const baseBackoffMs = opts?.baseBackoffMs ?? 100;
|
||||
const maxBackoffMs = opts?.maxBackoffMs ?? 500;
|
||||
const jitterRatio = opts?.jitterRatio ?? 0.2;
|
||||
|
||||
let attempt = 1;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
if (this.shutdownStarted) throw error;
|
||||
if (
|
||||
attempt >= maxAttempts ||
|
||||
!MySQLDatabaseClient.isRetriableError(error)
|
||||
)
|
||||
throw error;
|
||||
|
||||
const raw = baseBackoffMs * 2 ** (attempt - 1);
|
||||
const capped = Math.min(maxBackoffMs, raw);
|
||||
const window = Math.round(capped * jitterRatio);
|
||||
const jitter =
|
||||
window === 0
|
||||
? 0
|
||||
: Math.floor(Math.random() * (window * 2 + 1)) - window;
|
||||
const delay = Math.max(0, capped + jitter);
|
||||
|
||||
console.warn(
|
||||
`[${label}] transient mysql error (${(error as { code?: string })?.code ?? 'unknown'}); retry ${attempt + 1}/${maxAttempts} in ${delay}ms`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
attempt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Internal pool lifecycle
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
@@ -28,20 +28,6 @@ import {
|
||||
type PostgresPoolClient,
|
||||
} from './PostgresDatabaseClient.js';
|
||||
|
||||
// Pass-through spy: the rewrite is pure, so its call count shows the memo.
|
||||
const { rewriteSpy } = vi.hoisted(() => ({ rewriteSpy: vi.fn() }));
|
||||
vi.mock('./preparePostgresSql.js', async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import('./preparePostgresSql.js')>();
|
||||
return {
|
||||
...actual,
|
||||
preparePostgresSql: (sql: string) => {
|
||||
rewriteSpy(sql);
|
||||
return actual.preparePostgresSql(sql);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
type QueryCall = {
|
||||
text: string;
|
||||
values?: unknown[];
|
||||
@@ -181,23 +167,6 @@ describe('PostgresDatabaseClient', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('rewrites a repeated query once', async () => {
|
||||
const pool = new RecordingPool();
|
||||
const client = new PostgresDatabaseClient(postgresConfig(), () => pool);
|
||||
await client.onServerStart();
|
||||
const query = 'SELECT `name` FROM `apps` WHERE `id` = ?';
|
||||
rewriteSpy.mockClear();
|
||||
|
||||
await client.read(query, [1]);
|
||||
await client.pread(query, [2]);
|
||||
await client.write(query, [3]);
|
||||
|
||||
expect(rewriteSpy).toHaveBeenCalledTimes(1);
|
||||
expect(pool.calls.slice(-3).map((c) => c.text)).toEqual(
|
||||
Array(3).fill('SELECT "name" FROM "apps" WHERE "id" = $1'),
|
||||
);
|
||||
});
|
||||
|
||||
it('normalizes int8 fields on read and primary read rows', async () => {
|
||||
const pool = new RecordingPool(
|
||||
new RecordingPoolClient(),
|
||||
|
||||
@@ -17,14 +17,16 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { isAbsolute, resolve as resolvePath } from 'path';
|
||||
import { Pool, type PoolConfig, type QueryResult } from 'pg';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
import { Span } from '../../util/span.js';
|
||||
import {
|
||||
AbstractDatabaseClient,
|
||||
type BatchEntry,
|
||||
type WriteResult,
|
||||
} from './DatabaseClient';
|
||||
import { compareMigrationFilenames } from './migrationFilenames.js';
|
||||
import { preparePostgresSql } from './preparePostgresSql.js';
|
||||
import { splitPostgresStatements } from './splitPostgresStatements.js';
|
||||
import type { IConfig } from '../../types';
|
||||
@@ -60,7 +62,6 @@ enum Configuration {
|
||||
}
|
||||
|
||||
const POSTGRES_INT8_OID = 20;
|
||||
const PREPARED_SQL_CACHE_SIZE = 1000;
|
||||
const INTEGER_TEXT_PATTERN = /^-?\d+$/u;
|
||||
|
||||
const normalizePostgresInt8 = (
|
||||
@@ -137,10 +138,6 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
|
||||
private replicaPool!: PostgresPool;
|
||||
private configuration = Configuration.SINGLE;
|
||||
private shutdownStarted = false;
|
||||
/** Query text as written → its Postgres rewrite, which is pure. */
|
||||
private readonly preparedSql = new BoundedTtlMap<string, string>({
|
||||
maxEntries: PREPARED_SQL_CACHE_SIZE,
|
||||
});
|
||||
|
||||
constructor(
|
||||
config: IConfig,
|
||||
@@ -167,11 +164,7 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
|
||||
this.configuration = Configuration.SINGLE;
|
||||
}
|
||||
|
||||
await this.applyMigrationPaths(
|
||||
() => this.primaryPool.connect(),
|
||||
splitPostgresStatements,
|
||||
{ transactional: true },
|
||||
);
|
||||
await this.runMigrations();
|
||||
}
|
||||
|
||||
override async onServerPrepareShutdown(): Promise<void> {
|
||||
@@ -255,8 +248,98 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
|
||||
}
|
||||
}
|
||||
|
||||
protected override hasReadReplica(): boolean {
|
||||
return this.configuration === Configuration.REPLICA;
|
||||
@Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
|
||||
override async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
if (this.configuration === Configuration.SINGLE) {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
const primaryPromise = this.query(this.primaryPool, query, params);
|
||||
try {
|
||||
const replicaResult = await this.query(
|
||||
this.replicaPool,
|
||||
query,
|
||||
params,
|
||||
);
|
||||
if (replicaResult.rows.length > 0) {
|
||||
primaryPromise.catch(() => {});
|
||||
return normalizePostgresRows(replicaResult);
|
||||
}
|
||||
} catch {
|
||||
// fall through to primary
|
||||
}
|
||||
|
||||
const primaryResult = await primaryPromise;
|
||||
return normalizePostgresRows(primaryResult);
|
||||
}
|
||||
|
||||
private async runMigrations(): Promise<void> {
|
||||
const paths = this.config.database?.migrationPaths;
|
||||
if (!paths || paths.length === 0) return;
|
||||
|
||||
const conn = await this.primaryPool.connect();
|
||||
try {
|
||||
for (const rawPath of paths) {
|
||||
const dir = isAbsolute(rawPath)
|
||||
? rawPath
|
||||
: resolvePath(process.cwd(), rawPath);
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = readdirSync(dir)
|
||||
.filter(
|
||||
(f) =>
|
||||
f.endsWith('.sql') && f.startsWith('postgres'),
|
||||
)
|
||||
.sort(compareMigrationFilenames);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`[postgres] migration path is unreadable: ${dir}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log(`[postgres] no migrations in ${dir}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[postgres] running migrations from ${dir}: ${files.length} file(s)`,
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = resolvePath(dir, file);
|
||||
const contents = readFileSync(filePath, 'utf8');
|
||||
const statements = splitPostgresStatements(contents);
|
||||
await conn.query('BEGIN');
|
||||
try {
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
try {
|
||||
await conn.query(statements[i]);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`[postgres] failed to apply ${file} at statement ${i}`,
|
||||
{ cause: e },
|
||||
);
|
||||
}
|
||||
}
|
||||
await conn.query('COMMIT');
|
||||
} catch (e) {
|
||||
await conn.query('ROLLBACK').catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
console.log(
|
||||
`[postgres] applied ${file} (${statements.length} statements)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
}
|
||||
|
||||
private createPool(dbConf: PostgresEndpointConfig): PostgresPool {
|
||||
@@ -283,12 +366,8 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<QueryResult> {
|
||||
let text = this.preparedSql.get(query);
|
||||
if (text === undefined) {
|
||||
text = preparePostgresSql(query).text;
|
||||
this.preparedSql.set(query, text);
|
||||
}
|
||||
return target.query(text, params);
|
||||
const prepared = preparePostgresSql(query);
|
||||
return target.query(prepared.text, params);
|
||||
}
|
||||
|
||||
private async closeCurrentPools(): Promise<void> {
|
||||
|
||||
@@ -76,7 +76,6 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
export class SQLBatcher {
|
||||
dbPool;
|
||||
maxTimeInQueue;
|
||||
maxInFlightImmediate;
|
||||
maxBatchSize;
|
||||
maxQueueSize;
|
||||
failureThreshold;
|
||||
@@ -85,9 +84,7 @@ export class SQLBatcher {
|
||||
readOnly;
|
||||
acquireTimeoutMs;
|
||||
queue = [];
|
||||
#flushTimer = null;
|
||||
#flushTimerIsImmediate = false;
|
||||
#inFlight;
|
||||
timeouts = [];
|
||||
#consecutiveFailures = 0;
|
||||
#lastFailureAt = 0;
|
||||
#metricAttrs;
|
||||
@@ -96,11 +93,6 @@ export class SQLBatcher {
|
||||
* @param {object} dbPool Mysql2 pool
|
||||
* @param {object} [opts]
|
||||
* @param {number} [opts.maxTimeInQueue] Ms an item may wait before flush
|
||||
* @param {number} [opts.maxInFlightImmediate] While fewer flushes than this
|
||||
* are in flight, a lone item flushes on the next turn instead of waiting
|
||||
* maxTimeInQueue; past it, items wait and coalesce to spare connections
|
||||
* @param {{ count: number }} [opts.inFlight] In-flight flush counter the
|
||||
* cap reads; batchers sharing a pool can share one
|
||||
* @param {number} [opts.maxBatchSize] Items coalesced per flush
|
||||
* @param {number} [opts.maxQueueSize] Drop-oldest high-water mark
|
||||
* @param {number} [opts.failureThreshold] Consecutive failures to open the
|
||||
@@ -119,8 +111,6 @@ export class SQLBatcher {
|
||||
dbPool,
|
||||
{
|
||||
maxTimeInQueue = 20,
|
||||
maxInFlightImmediate = 0,
|
||||
inFlight = { count: 0 },
|
||||
maxBatchSize = 50,
|
||||
maxQueueSize = DEFAULT_MAX_QUEUE_SIZE,
|
||||
failureThreshold = DEFAULT_FAILURE_THRESHOLD,
|
||||
@@ -132,8 +122,6 @@ export class SQLBatcher {
|
||||
) {
|
||||
this.dbPool = dbPool;
|
||||
this.maxTimeInQueue = maxTimeInQueue;
|
||||
this.maxInFlightImmediate = maxInFlightImmediate;
|
||||
this.#inFlight = inFlight;
|
||||
this.maxBatchSize = maxBatchSize;
|
||||
this.maxQueueSize = maxQueueSize;
|
||||
this.failureThreshold = failureThreshold;
|
||||
@@ -197,34 +185,17 @@ export class SQLBatcher {
|
||||
|
||||
if (this.queue.length >= this.maxBatchSize) {
|
||||
this.flush(this.queue.splice(0, this.maxBatchSize));
|
||||
} else if (this.#flushTimer === null) {
|
||||
this.#armFlushTimer();
|
||||
} else if (this.queue.length === 1) {
|
||||
this.timeouts.push(
|
||||
setTimeout(() => {
|
||||
this.flush(this.queue.splice(0, this.queue.length));
|
||||
}, this.maxTimeInQueue),
|
||||
);
|
||||
}
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
// A lone flush holds a pooled connection for a full round trip, so only
|
||||
// the first maxInFlightImmediate flushes skip the wait.
|
||||
#armFlushTimer() {
|
||||
const run = () => {
|
||||
this.#flushTimer = null;
|
||||
this.flush(this.queue.splice(0, this.queue.length));
|
||||
};
|
||||
this.#flushTimerIsImmediate =
|
||||
this.#inFlight.count < this.maxInFlightImmediate;
|
||||
this.#flushTimer = this.#flushTimerIsImmediate
|
||||
? setImmediate(run)
|
||||
: setTimeout(run, this.maxTimeInQueue);
|
||||
}
|
||||
|
||||
#clearFlushTimer() {
|
||||
if (this.#flushTimer === null) return;
|
||||
if (this.#flushTimerIsImmediate) clearImmediate(this.#flushTimer);
|
||||
else clearTimeout(this.#flushTimer);
|
||||
this.#flushTimer = null;
|
||||
}
|
||||
|
||||
// Bounded wait for a pooled connection. Without a bound, a stalled
|
||||
// database turns every flush into an indefinite hang — nothing fails,
|
||||
// so neither the breaker nor callers' own timeouts ever engage.
|
||||
@@ -281,23 +252,22 @@ export class SQLBatcher {
|
||||
}
|
||||
|
||||
async flush(batch) {
|
||||
this.#clearFlushTimer();
|
||||
const timeout = this.timeouts.shift();
|
||||
if (timeout && !timeout._destroyed) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
if (batch.length === 0) return;
|
||||
|
||||
this.#inFlight.count++;
|
||||
try {
|
||||
await this.#flushBatch(batch);
|
||||
} finally {
|
||||
this.#inFlight.count--;
|
||||
}
|
||||
}
|
||||
const query = `${batch.map((b) => b.sql.replace(/;+\s*$/, '')).join(';')}; SELECT 1`; // SELECT 1 forces mysql2 to return array
|
||||
const values = batch.map((b) => b.values ?? []).flat();
|
||||
|
||||
async #flushBatch(batch) {
|
||||
let connection;
|
||||
try {
|
||||
connection = await this.#acquireConnection();
|
||||
} catch (error) {
|
||||
this.#recordFlushFailure();
|
||||
this.#consecutiveFailures++;
|
||||
this.#lastFailureAt = Date.now();
|
||||
flushFailureCounter.add(1, this.#metricAttrs);
|
||||
console.warn(
|
||||
'SQLBatcher could not acquire connection for flush:',
|
||||
error,
|
||||
@@ -308,14 +278,6 @@ export class SQLBatcher {
|
||||
return;
|
||||
}
|
||||
|
||||
if (batch.length === 1 && !this.readOnly) {
|
||||
await this.#flushLoneWrite(batch[0], connection);
|
||||
return;
|
||||
}
|
||||
|
||||
const query = `${batch.map((b) => b.sql.replace(/;+\s*$/, '')).join(';')}; SELECT 1`; // SELECT 1 forces mysql2 to return array
|
||||
const values = batch.map((b) => b.values ?? []).flat();
|
||||
|
||||
// Run the coalesced multi-statement inside an explicit transaction so
|
||||
// a single bad statement (e.g. a duplicate-key INSERT) rolls back the
|
||||
// whole batch atomically, leaving us free to re-run each item
|
||||
@@ -371,7 +333,7 @@ export class SQLBatcher {
|
||||
async () => {
|
||||
while (cursor < batch.length) {
|
||||
const i = cursor++;
|
||||
settled[i] = await this.#runItem(batch[i]);
|
||||
settled[i] = await this.#runFallbackItem(batch[i]);
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -405,34 +367,7 @@ export class SQLBatcher {
|
||||
}
|
||||
}
|
||||
|
||||
// A single statement is atomic under autocommit, so it skips BEGIN/COMMIT
|
||||
// and their round trips. With no transaction to roll back, a failure is
|
||||
// not re-run wholesale; only the per-item retry policy applies.
|
||||
async #flushLoneWrite(item, connection) {
|
||||
const r = await this.#runItem(item, connection);
|
||||
if (r.ok) {
|
||||
this.#consecutiveFailures = 0;
|
||||
item.resolve(r.value);
|
||||
return;
|
||||
}
|
||||
// An error the server answered with (duplicate key, bad SQL) means it
|
||||
// is up; only failing to reach it counts toward the breaker.
|
||||
if (isRetriableError(r.error)) {
|
||||
this.#recordFlushFailure();
|
||||
} else {
|
||||
this.#consecutiveFailures = 0;
|
||||
flushFailureCounter.add(1, this.#metricAttrs);
|
||||
}
|
||||
item.reject(r.error);
|
||||
}
|
||||
|
||||
#recordFlushFailure() {
|
||||
this.#consecutiveFailures++;
|
||||
this.#lastFailureAt = Date.now();
|
||||
flushFailureCounter.add(1, this.#metricAttrs);
|
||||
}
|
||||
|
||||
// Run one item on its own, retrying transient failures with backoff.
|
||||
// Run one fallback item, retrying transient failures with backoff.
|
||||
// A read-only batcher may retry anything transient; a batcher that
|
||||
// carries writes only retries failures where the statement provably
|
||||
// did not apply — either it never reached the server, or the server
|
||||
@@ -443,37 +378,33 @@ export class SQLBatcher {
|
||||
// than surfacing: an item is a single statement, so a deadlock victim has
|
||||
// been fully undone, and the caller sees an unhandled 500 for what the
|
||||
// database is telling us to just run again.
|
||||
async #runItem(b, connection) {
|
||||
async #runFallbackItem(b) {
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
if (!connection) {
|
||||
try {
|
||||
connection = await this.#acquireConnection();
|
||||
} catch (error) {
|
||||
return { ok: false, error };
|
||||
}
|
||||
let connection;
|
||||
try {
|
||||
connection = await this.#acquireConnection();
|
||||
} catch (error) {
|
||||
return { ok: false, error };
|
||||
}
|
||||
let error;
|
||||
try {
|
||||
return {
|
||||
ok: true,
|
||||
value: await connection.query(b.sql, b.values ?? []),
|
||||
};
|
||||
} catch (e) {
|
||||
error = e;
|
||||
} catch (error) {
|
||||
const canRetry = this.readOnly
|
||||
? isRetriableError(error) || isRolledBackError(error)
|
||||
: isNeverSentError(error) || isRolledBackError(error);
|
||||
if (!canRetry || attempt >= ITEM_RETRY_ATTEMPTS) {
|
||||
return { ok: false, error };
|
||||
}
|
||||
attempt++;
|
||||
fallbackItemRetriesCounter.add(1, this.#metricAttrs);
|
||||
await sleep(RETRY_BASE_BACKOFF_MS * attempt);
|
||||
} finally {
|
||||
connection.release();
|
||||
connection = undefined;
|
||||
}
|
||||
const canRetry = this.readOnly
|
||||
? isRetriableError(error) || isRolledBackError(error)
|
||||
: isNeverSentError(error) || isRolledBackError(error);
|
||||
if (!canRetry || attempt >= ITEM_RETRY_ATTEMPTS) {
|
||||
return { ok: false, error };
|
||||
}
|
||||
attempt++;
|
||||
fallbackItemRetriesCounter.add(1, this.#metricAttrs);
|
||||
await sleep(RETRY_BASE_BACKOFF_MS * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,356 +69,6 @@ const happyBatch = (sql: string) => {
|
||||
];
|
||||
};
|
||||
|
||||
// Each query on this connection waits until the test releases it, so its
|
||||
// flush stays in flight.
|
||||
const makeHeldConnection = () => {
|
||||
const held: Array<() => void> = [];
|
||||
const conn = makeConnection(
|
||||
(sql) =>
|
||||
new Promise((resolve) => {
|
||||
held.push(() => resolve(happyBatch(sql)));
|
||||
}),
|
||||
);
|
||||
const releaseAll = () => {
|
||||
for (const release of held.splice(0)) release();
|
||||
};
|
||||
const sent = () => conn.query.mock.calls.map(([sql]) => sql);
|
||||
return { conn, releaseAll, sent };
|
||||
};
|
||||
|
||||
const fakeFlushTimers = () =>
|
||||
vi.useFakeTimers({
|
||||
toFake: [
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
|
||||
describe('SQLBatcher flush timing', () => {
|
||||
it('flushes a lone query on the next turn while under the in-flight cap', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 60_000,
|
||||
maxInFlightImmediate: 2,
|
||||
});
|
||||
|
||||
const pending = batcher.query('SELECT a', []);
|
||||
expect(getConnection).not.toHaveBeenCalled();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(getConnection).toHaveBeenCalledTimes(1);
|
||||
expect((await pending)[0]).toEqual([{ n: 0 }]);
|
||||
});
|
||||
|
||||
it('holds a lone query for maxTimeInQueue without an in-flight cap', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
});
|
||||
|
||||
const pending = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
expect(getConnection).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(getConnection).toHaveBeenCalledTimes(1);
|
||||
expect((await pending)[0]).toEqual([{ n: 0 }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('batches queries from the same turn and sends later ones separately', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxInFlightImmediate: 8,
|
||||
});
|
||||
|
||||
const sameTurn = [
|
||||
batcher.query('SELECT a', []),
|
||||
batcher.query('SELECT b', []),
|
||||
batcher.query('SELECT c', []),
|
||||
];
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const nextTurn = batcher.query('SELECT d', []);
|
||||
const results = await Promise.all([...sameTurn, nextTurn]);
|
||||
|
||||
expect(results.map((r) => r[0])).toEqual([
|
||||
[{ n: 0 }],
|
||||
[{ n: 1 }],
|
||||
[{ n: 2 }],
|
||||
[{ n: 0 }],
|
||||
]);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
expect(conn.query.mock.calls.map(([sql]) => sql)).toEqual([
|
||||
'SELECT a;SELECT b;SELECT c; SELECT 1',
|
||||
'SELECT d; SELECT 1',
|
||||
]);
|
||||
});
|
||||
|
||||
it('gives a query enqueued during an in-flight flush its own flush while under the cap', async () => {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxInFlightImmediate: 2,
|
||||
});
|
||||
const turn = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await turn();
|
||||
const b = batcher.query('SELECT b', []);
|
||||
await turn();
|
||||
expect(sent()).toEqual(['SELECT a; SELECT 1', 'SELECT b; SELECT 1']);
|
||||
|
||||
releaseAll();
|
||||
expect((await a)[0]).toEqual([{ n: 0 }]);
|
||||
expect((await b)[0]).toEqual([{ n: 0 }]);
|
||||
});
|
||||
|
||||
it('holds a lone query for maxTimeInQueue at the in-flight cap and coalesces what follows', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toEqual(['SELECT a; SELECT 1']);
|
||||
|
||||
const b = batcher.query('SELECT b', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
const c = batcher.query('SELECT c', []);
|
||||
expect(sent()).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT b;SELECT c; SELECT 1',
|
||||
]);
|
||||
|
||||
releaseAll();
|
||||
expect((await a)[0]).toEqual([{ n: 0 }]);
|
||||
expect((await b)[0]).toEqual([{ n: 0 }]);
|
||||
expect((await c)[0]).toEqual([{ n: 1 }]);
|
||||
|
||||
// Settled flushes free the cap again.
|
||||
const d = batcher.query('SELECT d', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toHaveLength(3);
|
||||
releaseAll();
|
||||
expect((await d)[0]).toEqual([{ n: 0 }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('shares the in-flight cap between batchers given one counter', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool } = makePool(conn);
|
||||
const inFlight = { count: 0 };
|
||||
const opts = {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
inFlight,
|
||||
};
|
||||
const first = new SQLBatcher(pool, opts);
|
||||
const second = new SQLBatcher(pool, opts);
|
||||
const own = new SQLBatcher(pool, { ...opts, inFlight: undefined });
|
||||
|
||||
const a = first.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
const b = second.query('SELECT b', []);
|
||||
const c = own.query('SELECT c', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT c; SELECT 1',
|
||||
]);
|
||||
await vi.advanceTimersByTimeAsync(30);
|
||||
expect(sent()).toHaveLength(3);
|
||||
expect(sent()[2]).toBe('SELECT b; SELECT 1');
|
||||
|
||||
releaseAll();
|
||||
await Promise.all([a, b, c]);
|
||||
expect(inFlight.count).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('counts a flush as in flight until its per-item fallback settles', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const held: Array<() => void> = [];
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw makeError('ECONNRESET');
|
||||
return new Promise((resolve) => {
|
||||
held.push(() => resolve([[{ ok: sql }], undefined]));
|
||||
});
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const sent = () => conn.query.mock.calls.map(([sql]) => sql);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
acquireTimeoutMs: 0,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
// The batch failed; the per-item retry of `a` is still running.
|
||||
expect(sent()).toEqual(['SELECT a; SELECT 1', 'SELECT a']);
|
||||
|
||||
const b = batcher.query('SELECT b', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
expect(sent()).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT a',
|
||||
'SELECT b; SELECT 1',
|
||||
'SELECT b',
|
||||
]);
|
||||
|
||||
for (const release of held.splice(0)) release();
|
||||
expect((await a)[0]).toEqual([{ ok: 'SELECT a' }]);
|
||||
expect((await b)[0]).toEqual([{ ok: 'SELECT b' }]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('flushes a full batch at once past the cap and cancels the armed wait', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const { conn, releaseAll, sent } = makeHeldConnection();
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
readOnly: true,
|
||||
maxBatchSize: 2,
|
||||
maxTimeInQueue: 30,
|
||||
maxInFlightImmediate: 1,
|
||||
acquireTimeoutMs: 0,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
const b = batcher.query('SELECT b', []);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
const c = batcher.query('SELECT c', []);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
// A later lone query waits its own full maxTimeInQueue.
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
const d = batcher.query('SELECT d', []);
|
||||
await vi.advanceTimersByTimeAsync(29);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(getConnection).toHaveBeenCalledTimes(3);
|
||||
expect(sent()).toEqual([
|
||||
'SELECT a; SELECT 1',
|
||||
'SELECT b;SELECT c; SELECT 1',
|
||||
'SELECT d; SELECT 1',
|
||||
]);
|
||||
|
||||
releaseAll();
|
||||
await Promise.all([a, b, c, d]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('cancels the pending next-turn flush when a batch fills up', async () => {
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxBatchSize: 2,
|
||||
readOnly: true,
|
||||
maxInFlightImmediate: 8,
|
||||
acquireTimeoutMs: 0,
|
||||
});
|
||||
|
||||
const a = batcher.query('SELECT a', []);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
const b = batcher.query('SELECT b', []);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
const c = batcher.query('SELECT c', []);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
await vi.runAllTimersAsync();
|
||||
await Promise.all([a, b, c]);
|
||||
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves no timer armed after a failed flush, a drop or a breaker rejection', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
fakeFlushTimers();
|
||||
try {
|
||||
const getConnection = vi.fn(async () => {
|
||||
throw makeError('ECONNREFUSED');
|
||||
});
|
||||
const batcher = new SQLBatcher(
|
||||
{ promise: () => ({ getConnection }) },
|
||||
{
|
||||
readOnly: true,
|
||||
maxQueueSize: 1,
|
||||
failureThreshold: 1,
|
||||
cooldownMs: 60_000,
|
||||
maxInFlightImmediate: 8,
|
||||
},
|
||||
);
|
||||
|
||||
const dropped = expect(
|
||||
batcher.query('SELECT a', []),
|
||||
).rejects.toMatchObject({ reason: 'queueOverflow' });
|
||||
const failed = expect(
|
||||
batcher.query('SELECT b', []),
|
||||
).rejects.toMatchObject({ reason: 'connAcquire' });
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
await vi.runAllTimersAsync();
|
||||
await dropped;
|
||||
await failed;
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
await expect(batcher.query('SELECT c', [])).rejects.toMatchObject({
|
||||
reason: 'breakerOpen',
|
||||
});
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(batcher.queue).toHaveLength(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SQLBatcher', () => {
|
||||
it('resolves each batched item with its own result', async () => {
|
||||
const conn = makeConnection(happyBatch);
|
||||
@@ -476,7 +126,7 @@ describe('SQLBatcher', () => {
|
||||
});
|
||||
|
||||
it('drops the oldest item with reason queueOverflow at the high-water mark', async () => {
|
||||
const conn = makeConnection(() => [[{ n: 0 }], undefined]);
|
||||
const conn = makeConnection(happyBatch);
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxTimeInQueue: 5,
|
||||
@@ -627,65 +277,6 @@ describe('SQLBatcher', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('sends a lone write as-is, outside a transaction', async () => {
|
||||
const conn = makeConnection((sql) => {
|
||||
if (isBatchQuery(sql)) throw new Error('unexpected batch query');
|
||||
return [{ affectedRows: 1 }, undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxBatchSize: 1 });
|
||||
|
||||
const result = await batcher.query('UPDATE t SET x = ?', [1]);
|
||||
expect(result[0]).toEqual({ affectedRows: 1 });
|
||||
expect(conn.query).toHaveBeenCalledTimes(1);
|
||||
expect(conn.query).toHaveBeenCalledWith('UPDATE t SET x = ?', [1]);
|
||||
expect(conn.beginTransaction).not.toHaveBeenCalled();
|
||||
expect(conn.commit).not.toHaveBeenCalled();
|
||||
expect(conn.release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('runs a lone write the server rejects once, without tripping the breaker', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const conn = makeConnection((sql) => {
|
||||
if (sql === 'INSERT dup') throw makeError('ER_DUP_ENTRY');
|
||||
return [{ affectedRows: 1 }, undefined];
|
||||
});
|
||||
const { pool } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, {
|
||||
maxBatchSize: 1,
|
||||
failureThreshold: 1,
|
||||
cooldownMs: 60_000,
|
||||
});
|
||||
|
||||
await expect(batcher.query('INSERT dup', [])).rejects.toMatchObject({
|
||||
code: 'ER_DUP_ENTRY',
|
||||
});
|
||||
expect(conn.query).toHaveBeenCalledTimes(1);
|
||||
expect(conn.rollback).not.toHaveBeenCalled();
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
// The server answered, so it is up: the next write goes through.
|
||||
await expect(batcher.query('INSERT fine', [])).resolves.toBeTruthy();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('retries a lone write on a connection the server closed for inactivity', async () => {
|
||||
let attempts = 0;
|
||||
const conn = makeConnection(() => {
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
throw makeError('ER_CLIENT_INTERACTION_TIMEOUT');
|
||||
}
|
||||
return [{ affectedRows: 1 }, undefined];
|
||||
});
|
||||
const { pool, getConnection } = makePool(conn);
|
||||
const batcher = new SQLBatcher(pool, { maxBatchSize: 1 });
|
||||
|
||||
const result = await batcher.query('INSERT x', []);
|
||||
expect(result[0]).toEqual({ affectedRows: 1 });
|
||||
expect(attempts).toBe(2);
|
||||
expect(getConnection).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('never retries deterministic row-level errors and does not escalate the breaker', async () => {
|
||||
let fallbackAttempts = 0;
|
||||
const conn = makeConnection((sql) => {
|
||||
|
||||
@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
|
||||
import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
|
||||
|
||||
/** Highest schema version the migration table can reach. */
|
||||
const CURRENT_SCHEMA_VERSION = 91;
|
||||
const CURRENT_SCHEMA_VERSION = 82;
|
||||
|
||||
/**
|
||||
* These suites migrate real files on disk. Idle they finish in well under a
|
||||
@@ -930,7 +930,6 @@ describe('SqliteDatabaseClient — query interface', () => {
|
||||
affectedRows: 1,
|
||||
anyRowsAffected: true,
|
||||
});
|
||||
expect(inserted).not.toHaveProperty('warningStatus');
|
||||
|
||||
const missed = await client.write(
|
||||
'UPDATE `widget` SET `name` = ? WHERE `id` = ?',
|
||||
@@ -1039,7 +1038,8 @@ describe('SqliteDatabaseClient — query interface', () => {
|
||||
});
|
||||
|
||||
// Single-node engines have no replica to race, so `tryHardRead` must not
|
||||
// issue the query twice on the one connection.
|
||||
// issue the query twice — the base-class default fires `pread()` and
|
||||
// `read()` in parallel, which on sqlite is the same connection.
|
||||
describe('SqliteDatabaseClient — tryHardRead', () => {
|
||||
let client: SqliteDatabaseClient;
|
||||
|
||||
@@ -1078,50 +1078,20 @@ describe('SqliteDatabaseClient — tryHardRead', () => {
|
||||
expect(readSpy).toHaveBeenCalledTimes(1);
|
||||
expect(preadSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('SqliteDatabaseClient — prepared statements', () => {
|
||||
let client: SqliteDatabaseClient;
|
||||
let prepare: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(async () => {
|
||||
client = await bootClient();
|
||||
await client.write(
|
||||
'CREATE TABLE `widget` (`id` INTEGER PRIMARY KEY, `name` TEXT)',
|
||||
);
|
||||
// better-sqlite3 is the driver boundary; count compilations there.
|
||||
const db = (client as unknown as { db: Database.Database }).db;
|
||||
prepare = vi.spyOn(db, 'prepare');
|
||||
it('throws from requireRead when nothing matches', async () => {
|
||||
await expect(
|
||||
client.requireRead(
|
||||
'SELECT `name` FROM `widget` WHERE `id` = ?',
|
||||
[42],
|
||||
),
|
||||
).rejects.toThrow('required read returned no rows');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
client.onServerShutdown();
|
||||
});
|
||||
|
||||
it('compiles a repeated query once', async () => {
|
||||
const insert = 'INSERT INTO `widget` (`name`) VALUES (?)';
|
||||
await client.write(insert, ['a']);
|
||||
await client.write(insert, ['b']);
|
||||
await client.batchWrite([{ statement: insert, values: ['c'] }]);
|
||||
const select = 'SELECT `name` FROM `widget` ORDER BY `id`';
|
||||
await client.read(select);
|
||||
|
||||
await expect(client.read(select)).resolves.toEqual([
|
||||
{ name: 'a' },
|
||||
{ name: 'b' },
|
||||
{ name: 'c' },
|
||||
]);
|
||||
expect(prepare).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('keeps serving a cached query after the schema changes', async () => {
|
||||
const select = 'SELECT * FROM `widget`';
|
||||
await client.write('INSERT INTO `widget` (`name`) VALUES (?)', ['a']);
|
||||
await client.read(select);
|
||||
await client.write('ALTER TABLE `widget` ADD COLUMN `size` INTEGER');
|
||||
|
||||
await expect(client.read(select)).resolves.toEqual([
|
||||
{ id: 1, name: 'a', size: null },
|
||||
]);
|
||||
it('returns the rows from requireRead when something matches', async () => {
|
||||
await client.write('INSERT INTO `widget` (`name`) VALUES (?)', ['ok']);
|
||||
await expect(
|
||||
client.requireRead('SELECT `name` FROM `widget`'),
|
||||
).resolves.toEqual([{ name: 'ok' }]);
|
||||
});
|
||||
});
|
||||
@@ -21,15 +21,10 @@ import { existsSync, mkdirSync, readFileSync } from 'fs';
|
||||
import { basename, dirname, extname, join, resolve } from 'path';
|
||||
import { createContext, runInContext } from 'vm';
|
||||
import type { IConfig } from '../../types';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
import { Span } from '../../util/span.js';
|
||||
import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
|
||||
|
||||
const MIGRATIONS_DIR = resolve(__dirname, './migrations/sqlite');
|
||||
const STATEMENT_CACHE_SIZE = 1000;
|
||||
|
||||
type SqliteDatabase = InstanceType<typeof import('better-sqlite3')>;
|
||||
type SqliteStatement = import('better-sqlite3').Statement<unknown[]>;
|
||||
|
||||
/**
|
||||
* Ordered list of [threshold_version, files[]] pairs. A database whose
|
||||
@@ -121,26 +116,13 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
|
||||
[79, ['0084_share-anyone-with-link.sql']],
|
||||
[80, ['0085_event-subscriptions-include-value.sql']],
|
||||
[81, ['0086_subdomains-app-owner-cascade.sql']],
|
||||
[82, ['0087_feedback-attachments.sql']],
|
||||
[83, ['0088_apps-index-url.sql']],
|
||||
[84, ['0089_team-require-2fa.sql']],
|
||||
[85, ['0090_user-app-permission-index.sql']],
|
||||
[86, ['0091_icloud-dot-significance.sql']],
|
||||
[87, ['0092_share-entry-owner.sql']],
|
||||
[88, ['0093_fsentries-parent-uid-index.sql']],
|
||||
[89, ['0094_deleted-apps.sql']],
|
||||
[90, ['0095_kv-share-handle-levels.sql']],
|
||||
];
|
||||
|
||||
export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
override readonly engineName = 'sqlite';
|
||||
|
||||
// better-sqlite3 instance — set during onServerStart
|
||||
private db!: SqliteDatabase;
|
||||
/** Prepared statements by query text; SQLite re-prepares on schema change. */
|
||||
private readonly statements = new BoundedTtlMap<string, SqliteStatement>({
|
||||
maxEntries: STATEMENT_CACHE_SIZE,
|
||||
});
|
||||
private db!: InstanceType<typeof import('better-sqlite3')>;
|
||||
|
||||
constructor(config: IConfig) {
|
||||
super(config);
|
||||
@@ -162,14 +144,12 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
mkdirSync(dirname(dbPath), { recursive: true });
|
||||
}
|
||||
|
||||
this.statements.clear();
|
||||
this.db = new Database(dbPath);
|
||||
|
||||
await this.runMigrations(isNew);
|
||||
}
|
||||
|
||||
override onServerShutdown(): void {
|
||||
this.statements.clear();
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
@@ -184,9 +164,12 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
return this.prepare(query).all(
|
||||
...this.transformParams(params),
|
||||
) as Record<string, unknown>[];
|
||||
query = this.transformQuery(query);
|
||||
params = this.transformParams(params);
|
||||
return this.db.prepare(query).all(...params) as Record<
|
||||
string,
|
||||
unknown
|
||||
>[];
|
||||
}
|
||||
|
||||
override async pread(
|
||||
@@ -197,20 +180,30 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
override async tryHardRead(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
// No replica to race, so the base class's parallel
|
||||
// primary-plus-replica read would run the same statement twice on
|
||||
// the one connection.
|
||||
return this.read(query, params);
|
||||
}
|
||||
|
||||
@Span('db.write', (query: string) => ({ 'db.statement': query }))
|
||||
override async write(
|
||||
query: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<WriteResult> {
|
||||
const info = this.prepare(query).run(...this.transformParams(params));
|
||||
query = this.transformQuery(query);
|
||||
params = this.transformParams(params);
|
||||
|
||||
const info = this.db.prepare(query).run(...params);
|
||||
|
||||
return {
|
||||
insertId: info.lastInsertRowid,
|
||||
affectedRows: info.changes,
|
||||
anyRowsAffected: info.changes > 0,
|
||||
// No warningStatus: SQLite raises none, but a new row can reuse a
|
||||
// deleted row's rowid that other tables still name, so a clean
|
||||
// insert doesn't prove the row is all the caller sent.
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,8 +214,10 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
entries: { statement: string; values: unknown[] }[],
|
||||
): Promise<void> {
|
||||
this.db.transaction(() => {
|
||||
for (const { statement, values } of entries) {
|
||||
this.prepare(statement).run(...this.transformParams(values));
|
||||
for (let { statement, values } of entries) {
|
||||
statement = this.transformQuery(statement);
|
||||
values = this.transformParams(values);
|
||||
this.db.prepare(statement).run(...values);
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -231,15 +226,6 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
|
||||
// SQLite-specific transforms
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private prepare(query: string): SqliteStatement {
|
||||
let statement = this.statements.get(query);
|
||||
if (!statement) {
|
||||
statement = this.db.prepare(this.transformQuery(query));
|
||||
this.statements.set(query, statement);
|
||||
}
|
||||
return statement;
|
||||
}
|
||||
|
||||
private transformQuery(query: string): string {
|
||||
return query.replace(/now\(\)/gi, "datetime('now')");
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0087: Contact Us attachment metadata, a JSON array
|
||||
-- of `{name, type, size}` (NULL when none). Guarded by a throwaway procedure,
|
||||
-- as in mysql_mig_21.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_feedback_attachments;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_feedback_attachments()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'feedback'
|
||||
AND COLUMN_NAME = 'attachments'
|
||||
) THEN
|
||||
ALTER TABLE `feedback` ADD COLUMN `attachments` text DEFAULT NULL;
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_feedback_attachments();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_feedback_attachments;
|
||||
@@ -1,44 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0088: index `apps.index_url` for origin lookups.
|
||||
-- TEXT needs a prefix; 191 characters keeps the key under 767 bytes in
|
||||
-- utf8mb4 whatever the table's row format, and URLs diverge well before it.
|
||||
--
|
||||
-- Idempotent: the guarded procedure, as mig_36. The 1061 handler covers two
|
||||
-- nodes booting at once, where the loser finds the index already built.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_apps_index_url_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_apps_index_url_index()
|
||||
BEGIN
|
||||
DECLARE CONTINUE HANDLER FOR 1061 BEGIN END;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'apps'
|
||||
AND INDEX_NAME = 'idx_apps_index_url'
|
||||
) THEN
|
||||
ALTER TABLE `apps`
|
||||
ADD INDEX `idx_apps_index_url` (`index_url`(191)),
|
||||
ALGORITHM=INPLACE, LOCK=NONE;
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_apps_index_url_index();
|
||||
DROP PROCEDURE IF EXISTS _puter_add_apps_index_url_index;
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0089: a team may require 2FA of the accounts it
|
||||
-- provisioned. Added through `_puter_add_col` as mysql_mig_32 does, so a
|
||||
-- replay finds the column and does nothing.
|
||||
CALL _puter_add_col('group', 'require_2fa', '`require_2fa` tinyint(1) NOT NULL DEFAULT 0');
|
||||
@@ -1,39 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0090. Guarded like mysql_mig_22, because there is
|
||||
-- no per-file applied-state tracking and a replay must do nothing.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_app_permission_index;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_user_app_permission_index()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'user_to_app_permissions'
|
||||
AND INDEX_NAME = 'idx_user_to_app_permissions_permission'
|
||||
) THEN
|
||||
ALTER TABLE `user_to_app_permissions`
|
||||
ADD INDEX `idx_user_to_app_permissions_permission` (`permission`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_user_app_permission_index();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_user_app_permission_index;
|
||||
@@ -1,49 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Recompute `clean_email` for Apple's domains: dots are significant there, so
|
||||
-- rows written while they were stripped still collapse two mailboxes into one.
|
||||
-- Only rows still holding exactly what the old rule produced, so a NULL or a
|
||||
-- legacy un-lowercased value is left alone -- rewriting those would fold `+tag`
|
||||
-- or case into a key another row already owns. Restores dots and nothing else.
|
||||
|
||||
UPDATE `user`
|
||||
SET `clean_email` = CONCAT(SUBSTRING_INDEX(SUBSTRING_INDEX(LOWER(`email`), '@', 1), '+', 1), '@', SUBSTRING_INDEX(LOWER(`email`), '@', -1))
|
||||
WHERE `email` IS NOT NULL
|
||||
AND LOCATE('@', `email`) > 1
|
||||
AND (
|
||||
LOWER(`email`) LIKE '%@icloud.com'
|
||||
OR LOWER(`email`) LIKE '%@me.com'
|
||||
OR LOWER(`email`) LIKE '%@mac.com'
|
||||
)
|
||||
AND `clean_email` = CONCAT(REPLACE(SUBSTRING_INDEX(SUBSTRING_INDEX(LOWER(`email`), '@', 1), '+', 1), '.', ''), '@', SUBSTRING_INDEX(LOWER(`email`), '@', -1))
|
||||
AND `clean_email` <> CONCAT(SUBSTRING_INDEX(SUBSTRING_INDEX(LOWER(`email`), '@', 1), '+', 1), '@', SUBSTRING_INDEX(LOWER(`email`), '@', -1));
|
||||
|
||||
-- The same collapse decided who a pending invite belongs to, so the holder
|
||||
-- of the undotted address claims invites addressed to the dotted one. The
|
||||
-- typed address rides along as `invitedAddress` whenever it differed.
|
||||
UPDATE `share`
|
||||
SET `recipient_email` = CONCAT(SUBSTRING_INDEX(SUBSTRING_INDEX(LOWER(JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))), '@', 1), '+', 1), '@', SUBSTRING_INDEX(LOWER(JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))), '@', -1))
|
||||
WHERE JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress')) IS NOT NULL
|
||||
AND LOCATE('@', JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))) > 1
|
||||
AND (
|
||||
LOWER(`recipient_email`) LIKE '%@icloud.com'
|
||||
OR LOWER(`recipient_email`) LIKE '%@me.com'
|
||||
OR LOWER(`recipient_email`) LIKE '%@mac.com'
|
||||
)
|
||||
AND `recipient_email` = CONCAT(REPLACE(SUBSTRING_INDEX(SUBSTRING_INDEX(LOWER(JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))), '@', 1), '+', 1), '.', ''), '@', SUBSTRING_INDEX(LOWER(JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))), '@', -1))
|
||||
AND `recipient_email` <> CONCAT(SUBSTRING_INDEX(SUBSTRING_INDEX(LOWER(JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))), '@', 1), '+', 1), '@', SUBSTRING_INDEX(LOWER(JSON_UNQUOTE(JSON_EXTRACT(`data`, '$.invitedAddress'))), '@', -1));
|
||||
@@ -1,52 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- The owner of the shared entry, recorded on the share row. Mirrors SQLite
|
||||
-- migration 0092. Guarded like mysql_mig_45, because there is no per-file
|
||||
-- applied-state tracking and a replay must do nothing.
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_share_entry_owner;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_share_entry_owner()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND COLUMN_NAME = 'entry_owner_user_id'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD COLUMN `entry_owner_user_id` BIGINT DEFAULT NULL;
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'share'
|
||||
AND INDEX_NAME = 'idx_share_entry_owner'
|
||||
) THEN
|
||||
ALTER TABLE `share` ADD INDEX `idx_share_entry_owner` (`entry_owner_user_id`, `id`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_share_entry_owner();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_share_entry_owner;
|
||||
|
||||
UPDATE `share` JOIN `fsentries` ON `fsentries`.`id` = `share`.`fsentry_id`
|
||||
SET `share`.`entry_owner_user_id` = `fsentries`.`user_id`
|
||||
WHERE `share`.`fsentry_id` IS NOT NULL
|
||||
AND `share`.`entry_owner_user_id` IS NULL;
|
||||
@@ -1,26 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Uid and owner of each deleted app. Mirrors SQLite migration 0094.
|
||||
--
|
||||
-- Idempotent: `CREATE TABLE IF NOT EXISTS` lets the directory replay safely.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `deleted_apps` (
|
||||
`uid` VARCHAR(40) NOT NULL,
|
||||
`owner_user_id` INT UNSIGNED DEFAULT NULL,
|
||||
PRIMARY KEY (`uid`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -1,50 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Share levels on `kv_share_handles`. See
|
||||
-- sqlite/0095_kv-share-handle-levels.sql for the rationale. No per-file
|
||||
-- applied-state tracking, so every step is guarded and a replay does nothing.
|
||||
|
||||
CALL _puter_add_col('kv_share_handles', 'revoked_by', '`revoked_by` varchar(8) DEFAULT NULL');
|
||||
CALL _puter_add_col('kv_share_handles', 'label', '`label` varchar(512) DEFAULT NULL');
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_kv_share_handle_indexes;
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE _puter_add_kv_share_handle_indexes()
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_grantee'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_grantee` (`grantee_user_id`, `id`);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'kv_share_handles'
|
||||
AND INDEX_NAME = 'idx_kv_share_handles_app'
|
||||
) THEN
|
||||
ALTER TABLE `kv_share_handles` ADD INDEX `idx_kv_share_handles_app` (`app_uid`);
|
||||
END IF;
|
||||
END//
|
||||
DELIMITER ;
|
||||
|
||||
CALL _puter_add_kv_share_handle_indexes();
|
||||
|
||||
DROP PROCEDURE IF EXISTS _puter_add_kv_share_handle_indexes;
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0087: Contact Us attachment metadata, a JSON array
|
||||
-- of `{name, type, size}` (NULL when none).
|
||||
|
||||
ALTER TABLE feedback ADD COLUMN IF NOT EXISTS attachments text DEFAULT NULL;
|
||||
@@ -1,24 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0088: index `apps.index_url` for origin lookups.
|
||||
-- Hash rather than btree: `index_url` allows 3000 characters, past btree's
|
||||
-- ~2.7 KB row limit, and every lookup is equality (`=` / `IN`).
|
||||
--
|
||||
-- Idempotent via IF NOT EXISTS.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_apps_index_url ON apps USING hash (index_url);
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0089: a team may require 2FA of the accounts it
|
||||
-- provisioned. Idempotent -- there is no per-file applied-state tracking, so
|
||||
-- a replay must find the column already present and do nothing.
|
||||
ALTER TABLE "group" ADD COLUMN IF NOT EXISTS require_2fa smallint NOT NULL DEFAULT 0;
|
||||
@@ -1,24 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Mirrors SQLite migration 0090, as postgres_mig_11 did for user_to_user.
|
||||
-- text_pattern_ops is what makes the left-anchored LIKE a range scan under a
|
||||
-- non-C collation -- the plain index dev_to_app already has does not.
|
||||
CREATE INDEX IF NOT EXISTS idx_user_to_app_permissions_permission
|
||||
ON user_to_app_permissions (permission text_pattern_ops);
|
||||
CREATE INDEX IF NOT EXISTS idx_dev_to_app_permissions_permission_pattern
|
||||
ON dev_to_app_permissions (permission text_pattern_ops);
|
||||
@@ -1,49 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Recompute `clean_email` for Apple's domains: dots are significant there, so
|
||||
-- rows written while they were stripped still collapse two mailboxes into one.
|
||||
-- Only rows still holding exactly what the old rule produced, so a NULL or a
|
||||
-- legacy un-lowercased value is left alone -- rewriting those would fold `+tag`
|
||||
-- or case into a key another row already owns. Restores dots and nothing else.
|
||||
|
||||
UPDATE "user"
|
||||
SET clean_email = (split_part(split_part(lower(email), '@', 1), '+', 1) || '@' || split_part(lower(email), '@', 2))
|
||||
WHERE email IS NOT NULL
|
||||
AND position('@' in email) > 1
|
||||
AND (
|
||||
lower(email) LIKE '%@icloud.com'
|
||||
OR lower(email) LIKE '%@me.com'
|
||||
OR lower(email) LIKE '%@mac.com'
|
||||
)
|
||||
AND clean_email = (replace(split_part(split_part(lower(email), '@', 1), '+', 1), '.', '') || '@' || split_part(lower(email), '@', 2))
|
||||
AND clean_email IS DISTINCT FROM (split_part(split_part(lower(email), '@', 1), '+', 1) || '@' || split_part(lower(email), '@', 2));
|
||||
|
||||
-- The same collapse decided who a pending invite belongs to, so the holder
|
||||
-- of the undotted address claims invites addressed to the dotted one. The
|
||||
-- typed address rides along as `invitedAddress` whenever it differed.
|
||||
UPDATE "share"
|
||||
SET recipient_email = (split_part(split_part(lower((data->>'invitedAddress')), '@', 1), '+', 1) || '@' || split_part(lower((data->>'invitedAddress')), '@', 2))
|
||||
WHERE (data->>'invitedAddress') IS NOT NULL
|
||||
AND position('@' in (data->>'invitedAddress')) > 1
|
||||
AND (
|
||||
lower(recipient_email) LIKE '%@icloud.com'
|
||||
OR lower(recipient_email) LIKE '%@me.com'
|
||||
OR lower(recipient_email) LIKE '%@mac.com'
|
||||
)
|
||||
AND recipient_email = (replace(split_part(split_part(lower((data->>'invitedAddress')), '@', 1), '+', 1), '.', '') || '@' || split_part(lower((data->>'invitedAddress')), '@', 2))
|
||||
AND recipient_email IS DISTINCT FROM (split_part(split_part(lower((data->>'invitedAddress')), '@', 1), '+', 1) || '@' || split_part(lower((data->>'invitedAddress')), '@', 2));
|
||||
@@ -1,32 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- The owner of the shared entry, recorded on the share row. Mirrors SQLite
|
||||
-- migration 0092. Guarded like mysql_mig_45, because there is no per-file
|
||||
-- applied-state tracking and a replay must do nothing.
|
||||
|
||||
ALTER TABLE "share" ADD COLUMN IF NOT EXISTS entry_owner_user_id BIGINT DEFAULT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_share_entry_owner
|
||||
ON "share" (entry_owner_user_id, id);
|
||||
|
||||
UPDATE "share"
|
||||
SET entry_owner_user_id = f.user_id
|
||||
FROM fsentries f
|
||||
WHERE f.id = "share".fsentry_id
|
||||
AND "share".fsentry_id IS NOT NULL
|
||||
AND "share".entry_owner_user_id IS NULL;
|
||||
@@ -1,26 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Uid and owner of each deleted app. Mirrors SQLite migration 0094 / MySQL
|
||||
-- mysql_mig_48.
|
||||
--
|
||||
-- Idempotent via IF NOT EXISTS.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deleted_apps (
|
||||
uid VARCHAR(40) PRIMARY KEY,
|
||||
owner_user_id INTEGER DEFAULT NULL
|
||||
);
|
||||
@@ -1,29 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Share levels on kv_share_handles. See
|
||||
-- sqlite/0095_kv-share-handle-levels.sql for the rationale. Idempotent via
|
||||
-- IF NOT EXISTS; there is no per-file applied-state tracking.
|
||||
|
||||
ALTER TABLE kv_share_handles ADD COLUMN IF NOT EXISTS revoked_by VARCHAR(8) DEFAULT NULL;
|
||||
ALTER TABLE kv_share_handles ADD COLUMN IF NOT EXISTS label VARCHAR(512) DEFAULT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_grantee
|
||||
ON kv_share_handles (grantee_user_id, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kv_share_handles_app
|
||||
ON kv_share_handles (app_uid);
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Contact Us attachment metadata: a JSON array of `{name, type, size}`, NULL
|
||||
-- when none. The files themselves go out with the support email.
|
||||
|
||||
ALTER TABLE `feedback` ADD COLUMN "attachments" TEXT DEFAULT NULL;
|
||||
@@ -1,22 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Resolving an origin to its app looks `apps` up by `index_url` on every
|
||||
-- origin-based app token request (`AuthService#findCanonicalAppUidForOrigin`),
|
||||
-- plus `AppStore.existsByIndexUrl`. With no index each lookup scans the table.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_apps_index_url` ON `apps` (`index_url`);
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Whether every account this team provisioned must hold 2FA before it can
|
||||
-- use Puter. Off until the owner turns it on, and the owner must hold 2FA
|
||||
-- themselves first, or enabling it would lock out the account that did so.
|
||||
ALTER TABLE `group` ADD COLUMN `require_2fa` INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -1,26 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Withdrawing an app's cross-app data grants looks them up by permission text,
|
||||
-- and no existing index on either table leads with `permission`. Same fix as
|
||||
-- `idx_user_to_user_permissions_permission` in 0067.
|
||||
CREATE INDEX IF NOT EXISTS `idx_user_to_app_permissions_permission`
|
||||
ON `user_to_app_permissions` (`permission`);
|
||||
|
||||
-- The sweep reads both tables; MySQL and Postgres already index this one.
|
||||
CREATE INDEX IF NOT EXISTS `idx_dev_to_app_permissions_permission`
|
||||
ON `dev_to_app_permissions` (`permission`);
|
||||
@@ -1,73 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Recompute `clean_email` for Apple's domains: dots are significant there, so
|
||||
-- rows written while they were stripped still collapse two mailboxes into one.
|
||||
-- Only rows still holding exactly what the old rule produced, so a NULL or a
|
||||
-- legacy un-lowercased value is left alone -- rewriting those would fold `+tag`
|
||||
-- or case into a key another row already owns. Restores dots and nothing else.
|
||||
|
||||
UPDATE `user`
|
||||
SET `clean_email` = (CASE WHEN instr(substr(lower(`email`), 1, instr(`email`, '@') - 1), '+') > 0
|
||||
THEN substr(substr(lower(`email`), 1, instr(`email`, '@') - 1), 1,
|
||||
instr(substr(lower(`email`), 1, instr(`email`, '@') - 1), '+') - 1)
|
||||
ELSE substr(lower(`email`), 1, instr(`email`, '@') - 1)
|
||||
END || substr(lower(`email`), instr(`email`, '@')))
|
||||
WHERE `email` IS NOT NULL
|
||||
AND instr(`email`, '@') > 1
|
||||
AND (
|
||||
lower(`email`) LIKE '%@icloud.com'
|
||||
OR lower(`email`) LIKE '%@me.com'
|
||||
OR lower(`email`) LIKE '%@mac.com'
|
||||
)
|
||||
AND `clean_email` = (replace(CASE WHEN instr(substr(lower(`email`), 1, instr(`email`, '@') - 1), '+') > 0
|
||||
THEN substr(substr(lower(`email`), 1, instr(`email`, '@') - 1), 1,
|
||||
instr(substr(lower(`email`), 1, instr(`email`, '@') - 1), '+') - 1)
|
||||
ELSE substr(lower(`email`), 1, instr(`email`, '@') - 1)
|
||||
END, '.', '') || substr(lower(`email`), instr(`email`, '@')))
|
||||
AND `clean_email` <> (CASE WHEN instr(substr(lower(`email`), 1, instr(`email`, '@') - 1), '+') > 0
|
||||
THEN substr(substr(lower(`email`), 1, instr(`email`, '@') - 1), 1,
|
||||
instr(substr(lower(`email`), 1, instr(`email`, '@') - 1), '+') - 1)
|
||||
ELSE substr(lower(`email`), 1, instr(`email`, '@') - 1)
|
||||
END || substr(lower(`email`), instr(`email`, '@')));
|
||||
|
||||
-- The same collapse decided who a pending invite belongs to, so the holder
|
||||
-- of the undotted address claims invites addressed to the dotted one. The
|
||||
-- typed address rides along as `invitedAddress` whenever it differed.
|
||||
UPDATE `share`
|
||||
SET `recipient_email` = (CASE WHEN instr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), '+') > 0
|
||||
THEN substr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), 1,
|
||||
instr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), '+') - 1)
|
||||
ELSE substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1)
|
||||
END || substr(lower(json_extract(`data`, '$.invitedAddress')), instr(json_extract(`data`, '$.invitedAddress'), '@')))
|
||||
WHERE json_extract(`data`, '$.invitedAddress') IS NOT NULL
|
||||
AND instr(json_extract(`data`, '$.invitedAddress'), '@') > 1
|
||||
AND (
|
||||
lower(`recipient_email`) LIKE '%@icloud.com'
|
||||
OR lower(`recipient_email`) LIKE '%@me.com'
|
||||
OR lower(`recipient_email`) LIKE '%@mac.com'
|
||||
)
|
||||
AND `recipient_email` = (replace(CASE WHEN instr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), '+') > 0
|
||||
THEN substr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), 1,
|
||||
instr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), '+') - 1)
|
||||
ELSE substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1)
|
||||
END, '.', '') || substr(lower(json_extract(`data`, '$.invitedAddress')), instr(json_extract(`data`, '$.invitedAddress'), '@')))
|
||||
AND `recipient_email` <> (CASE WHEN instr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), '+') > 0
|
||||
THEN substr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), 1,
|
||||
instr(substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1), '+') - 1)
|
||||
ELSE substr(lower(json_extract(`data`, '$.invitedAddress')), 1, instr(json_extract(`data`, '$.invitedAddress'), '@') - 1)
|
||||
END || substr(lower(json_extract(`data`, '$.invitedAddress')), instr(json_extract(`data`, '$.invitedAddress'), '@')));
|
||||
@@ -1,40 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- The owner of the shared entry, recorded on the share row.
|
||||
--
|
||||
-- The outbound listings answer "shares on things I own that someone else
|
||||
-- issued" by joining `share` to `fsentries` and filtering on the owner there.
|
||||
-- Neither side of that join bounds the work: driven from `fsentries` it walks
|
||||
-- every file the user owns, driven from `share` it walks every share until it
|
||||
-- has a page. `#outboundAppsSql` has no LIMIT at all.
|
||||
--
|
||||
-- Written with the row, and re-pointed when a move re-owns the subtree it
|
||||
-- names; `FSService.move` does that whenever the destination is someone
|
||||
-- else's tree.
|
||||
|
||||
ALTER TABLE `share` ADD COLUMN `entry_owner_user_id` INTEGER DEFAULT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_share_entry_owner`
|
||||
ON `share` (`entry_owner_user_id`, `id`);
|
||||
|
||||
UPDATE `share`
|
||||
SET `entry_owner_user_id` = (
|
||||
SELECT `user_id` FROM `fsentries` WHERE `fsentries`.`id` = `share`.`fsentry_id`
|
||||
)
|
||||
WHERE `fsentry_id` IS NOT NULL
|
||||
AND `entry_owner_user_id` IS NULL;
|
||||
@@ -1,22 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Directory listings and subtree walks (delete, copy) find children by
|
||||
-- `parent_uid`, the one link every row carries; with no index each lookup
|
||||
-- scans the table. MySQL and Postgres already index it.
|
||||
CREATE INDEX IF NOT EXISTS `idx_fsentries_parent_uid`
|
||||
ON `fsentries` (`parent_uid`);
|
||||
@@ -1,25 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- Uid and owner of each deleted app. An origin-derived uid comes back when the
|
||||
-- origin is used again; on a hosted subdomain now registered to someone else,
|
||||
-- AuthService moves on to a fresh uid so the deleted app's data stays put.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `deleted_apps` (
|
||||
"uid" TEXT PRIMARY KEY NOT NULL,
|
||||
"owner_user_id" INTEGER DEFAULT NULL
|
||||
);
|
||||
@@ -1,35 +0,0 @@
|
||||
-- Copyright (C) 2024-present Puter Technologies Inc.
|
||||
--
|
||||
-- This file is part of Puter.
|
||||
--
|
||||
-- Puter is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU Affero General Public License as published
|
||||
-- by the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU Affero General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU Affero General Public License
|
||||
-- along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
-- What share levels need from `kv_share_handles`. The level itself is not a
|
||||
-- column: it is the root of the stored `permission`.
|
||||
--
|
||||
-- - `revoked_by` : `owner`, `holder` or `system` (the namespace app was
|
||||
-- deleted). Null while live, and on rows retired before
|
||||
-- this existed.
|
||||
-- - `label` : optional owner-set text the holder is shown.
|
||||
-- - the grantee index serves the holder's listing and their unshare, and
|
||||
-- the app index retiring a deleted app's handles.
|
||||
|
||||
ALTER TABLE `kv_share_handles` ADD COLUMN `revoked_by` TEXT DEFAULT NULL;
|
||||
ALTER TABLE `kv_share_handles` ADD COLUMN `label` TEXT DEFAULT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_grantee`
|
||||
ON `kv_share_handles` (`grantee_user_id`, `id`);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS `idx_kv_share_handles_app`
|
||||
ON `kv_share_handles` (`app_uid`);
|
||||
@@ -32,7 +32,6 @@ const RETRIABLE_ERROR_CODES = new Set([
|
||||
'ENETUNREACH',
|
||||
'EAI_AGAIN',
|
||||
POOL_ACQUIRE_TIMEOUT,
|
||||
'ER_CLIENT_INTERACTION_TIMEOUT',
|
||||
]);
|
||||
|
||||
const RETRIABLE_ERROR_MESSAGES = [
|
||||
@@ -53,9 +52,6 @@ const NEVER_SENT_ERROR_CODES = new Set([
|
||||
'ENETUNREACH',
|
||||
'EAI_AGAIN',
|
||||
POOL_ACQUIRE_TIMEOUT,
|
||||
// The server closed the pooled connection for inactivity before the
|
||||
// statement arrived, so nothing ran.
|
||||
'ER_CLIENT_INTERACTION_TIMEOUT',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -20,17 +20,7 @@
|
||||
|
||||
import { createServer, type Server } from 'node:http';
|
||||
import type { AddressInfo } from 'node:net';
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
type MockInstance,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import type { IConfig } from '../../types';
|
||||
import { DDBClient } from './DDBClient';
|
||||
|
||||
@@ -152,83 +142,6 @@ describe('DDBClient — item operations', () => {
|
||||
expect(result.Item).toBeUndefined();
|
||||
});
|
||||
|
||||
it('deletes only while its condition holds', async () => {
|
||||
await client.put(TABLE, { pk: 'u6', sk: 'guarded', v: 1 });
|
||||
|
||||
await expect(
|
||||
client.del(
|
||||
TABLE,
|
||||
{ pk: 'u6', sk: 'guarded' },
|
||||
{
|
||||
condition: {
|
||||
expression: '#v = :wrong',
|
||||
names: { '#v': 'v' },
|
||||
values: { ':wrong': 2 },
|
||||
},
|
||||
},
|
||||
),
|
||||
).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' });
|
||||
expect(
|
||||
(await client.get(TABLE, { pk: 'u6', sk: 'guarded' })).Item,
|
||||
).toBeDefined();
|
||||
|
||||
await client.del(
|
||||
TABLE,
|
||||
{ pk: 'u6', sk: 'guarded' },
|
||||
{
|
||||
condition: {
|
||||
expression: '#v = :right',
|
||||
names: { '#v': 'v' },
|
||||
values: { ':right': 1 },
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(
|
||||
(await client.get(TABLE, { pk: 'u6', sk: 'guarded' })).Item,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('puts only while its condition holds', async () => {
|
||||
const absent = {
|
||||
expression: 'attribute_not_exists(#pk)',
|
||||
names: { '#pk': 'pk' },
|
||||
};
|
||||
await client.put(
|
||||
TABLE,
|
||||
{ pk: 'u7', sk: 'once', v: 1 },
|
||||
{
|
||||
condition: absent,
|
||||
},
|
||||
);
|
||||
await expect(
|
||||
client.put(
|
||||
TABLE,
|
||||
{ pk: 'u7', sk: 'once', v: 2 },
|
||||
{
|
||||
condition: absent,
|
||||
},
|
||||
),
|
||||
).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' });
|
||||
expect(
|
||||
(await client.get(TABLE, { pk: 'u7', sk: 'once' })).Item?.v,
|
||||
).toBe(1);
|
||||
|
||||
await client.put(
|
||||
TABLE,
|
||||
{ pk: 'u7', sk: 'once', v: 3 },
|
||||
{
|
||||
condition: {
|
||||
expression: '#v = :held',
|
||||
names: { '#v': 'v' },
|
||||
values: { ':held': 1 },
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(
|
||||
(await client.get(TABLE, { pk: 'u7', sk: 'once' })).Item?.v,
|
||||
).toBe(3);
|
||||
});
|
||||
|
||||
it('fetches keys from several tables in one batch', async () => {
|
||||
await client.put(TABLE, { pk: 'b1', sk: 'a', v: 1 });
|
||||
await client.put(TABLE, { pk: 'b1', sk: 'b', v: 2 });
|
||||
@@ -451,59 +364,6 @@ describe('DDBClient — expired item sweep', () => {
|
||||
expect((await client.get(table, { pk: 'fresh' })).Item).toBeDefined();
|
||||
expect((await client.get(table, { pk: 'eternal' })).Item).toBeDefined();
|
||||
});
|
||||
|
||||
it('keeps an item whose ttl is 0, which never expires', async () => {
|
||||
const client = new DDBClient(localConfig());
|
||||
const table = 'ttl-zero-items';
|
||||
const schema = {
|
||||
TableName: table,
|
||||
KeySchema: [{ AttributeName: 'pk', KeyType: 'HASH' as const }],
|
||||
AttributeDefinitions: [
|
||||
{ AttributeName: 'pk', AttributeType: 'S' as const },
|
||||
],
|
||||
ProvisionedThroughput: {
|
||||
ReadCapacityUnits: 5,
|
||||
WriteCapacityUnits: 5,
|
||||
},
|
||||
};
|
||||
|
||||
await client.createTableIfNotExists(schema, 'expireAt');
|
||||
await awaitTable(client, table, { pk: 'ready' });
|
||||
await client.put(table, { pk: 'zero', expireAt: 0 });
|
||||
|
||||
// Second call finds the table in use, so the sweep runs.
|
||||
await client.createTableIfNotExists(schema, 'expireAt');
|
||||
|
||||
expect((await client.get(table, { pk: 'zero' })).Item).toBeDefined();
|
||||
});
|
||||
|
||||
it('keeps an item whose ttl is not a number, which the sweep can never purge', async () => {
|
||||
const client = new DDBClient(localConfig());
|
||||
const table = 'ttl-nan-items';
|
||||
const schema = {
|
||||
TableName: table,
|
||||
KeySchema: [{ AttributeName: 'pk', KeyType: 'HASH' as const }],
|
||||
AttributeDefinitions: [
|
||||
{ AttributeName: 'pk', AttributeType: 'S' as const },
|
||||
],
|
||||
ProvisionedThroughput: {
|
||||
ReadCapacityUnits: 5,
|
||||
WriteCapacityUnits: 5,
|
||||
},
|
||||
};
|
||||
|
||||
await client.createTableIfNotExists(schema, 'expireAt');
|
||||
await awaitTable(client, table, { pk: 'ready' });
|
||||
const past = Math.floor(Date.now() / 1000) - 60;
|
||||
await client.put(table, { pk: 'stringTtl', expireAt: String(past) });
|
||||
|
||||
// Second call finds the table in use, so the sweep runs.
|
||||
await client.createTableIfNotExists(schema, 'expireAt');
|
||||
|
||||
expect(
|
||||
(await client.get(table, { pk: 'stringTtl' })).Item,
|
||||
).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DDBClient — expired item sweep on an empty table', () => {
|
||||
@@ -542,19 +402,6 @@ describe('DDBClient — credentialled configuration', () => {
|
||||
).toThrow('requires both `access_key` and `secret_key`');
|
||||
});
|
||||
|
||||
it('accepts the credential chain in place of keys', async () => {
|
||||
const client = new DDBClient({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
dynamo: {
|
||||
aws: { useCredentialChain: true, region: 'us-west-2' },
|
||||
endpoint: 'http://127.0.0.1:1',
|
||||
},
|
||||
} as unknown as IConfig);
|
||||
|
||||
await expect(client.recreateClient()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips table creation on a managed deployment', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const client = new DDBClient({
|
||||
@@ -827,391 +674,3 @@ describe('DDBClient — unprocessed batch keys', () => {
|
||||
).resolves.toEqual({ Responses: {}, ConsumedCapacity: [] });
|
||||
});
|
||||
});
|
||||
|
||||
// Answers by operation (X-Amz-Target) and records how many requests overlap, so
|
||||
// chunk concurrency and the TTL sweep's retries can be asserted.
|
||||
describe('DDBClient — batch write chunks and TTL sweep', () => {
|
||||
let server: Server;
|
||||
let endpoint: string;
|
||||
let handlers: Record<string, (body: Record<string, unknown>) => unknown>;
|
||||
let calls: string[];
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = createServer((req, res) => {
|
||||
let body = '';
|
||||
req.on('data', (chunk) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.on('end', () => {
|
||||
const op = String(req.headers['x-amz-target']).split('.')[1];
|
||||
calls.push(op);
|
||||
inFlight += 1;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
setTimeout(() => {
|
||||
inFlight -= 1;
|
||||
const reply = handlers[op]?.(JSON.parse(body)) ?? {};
|
||||
const { status = 200, ...payload } = reply as {
|
||||
status?: number;
|
||||
};
|
||||
res.writeHead(status, {
|
||||
'content-type': 'application/x-amz-json-1.0',
|
||||
});
|
||||
res.end(JSON.stringify(payload));
|
||||
}, 20);
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) =>
|
||||
server.listen(0, '127.0.0.1', resolve),
|
||||
);
|
||||
endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
calls = [];
|
||||
inFlight = 0;
|
||||
peakInFlight = 0;
|
||||
});
|
||||
|
||||
const stubClient = (extra: Record<string, unknown> = {}) =>
|
||||
new DDBClient({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
dynamo: {
|
||||
aws: { access_key: 'a', secret_key: 'b', region: 'us-west-2' },
|
||||
endpoint,
|
||||
...extra,
|
||||
},
|
||||
} as unknown as IConfig);
|
||||
|
||||
it('writes chunks concurrently, a bounded number at a time', async () => {
|
||||
handlers = { BatchWriteItem: () => ({ UnprocessedItems: {} }) };
|
||||
|
||||
await stubClient().batchPut(
|
||||
Array.from({ length: 200 }, (_, i) => ({
|
||||
table: TABLE,
|
||||
item: { pk: `item-${i}` },
|
||||
})),
|
||||
);
|
||||
|
||||
expect(calls).toHaveLength(8);
|
||||
expect(peakInFlight).toBeGreaterThan(1);
|
||||
expect(peakInFlight).toBeLessThanOrEqual(4);
|
||||
});
|
||||
|
||||
it('stops starting chunks once one fails', async () => {
|
||||
handlers = {
|
||||
BatchWriteItem: () => ({
|
||||
status: 400,
|
||||
__type: 'com.amazon.coral.validate#ValidationException',
|
||||
message: 'bad item',
|
||||
}),
|
||||
};
|
||||
|
||||
await expect(
|
||||
stubClient().batchPut(
|
||||
Array.from({ length: 250 }, (_, i) => ({
|
||||
table: TABLE,
|
||||
item: { pk: `item-${i}` },
|
||||
})),
|
||||
),
|
||||
).rejects.toThrow('bad item');
|
||||
// The first wave fails; none of the six chunks queued behind it run.
|
||||
expect(calls.length).toBeLessThanOrEqual(4);
|
||||
});
|
||||
|
||||
it('lets chunks already in flight finish before rejecting', async () => {
|
||||
let replies = 0;
|
||||
handlers = {
|
||||
BatchWriteItem: (body) => {
|
||||
replies += 1;
|
||||
if (replies === 1) {
|
||||
return {
|
||||
status: 400,
|
||||
__type: 'com.amazon.coral.validate#ValidationException',
|
||||
message: 'bad item',
|
||||
};
|
||||
}
|
||||
// The rest need one retry, so they are still writing when the
|
||||
// failure comes back.
|
||||
return replies <= 4
|
||||
? { UnprocessedItems: body.RequestItems }
|
||||
: { UnprocessedItems: {} };
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
stubClient().batchPut(
|
||||
Array.from({ length: 100 }, (_, i) => ({
|
||||
table: TABLE,
|
||||
item: { pk: `item-${i}` },
|
||||
})),
|
||||
),
|
||||
).rejects.toThrow('bad item');
|
||||
const sentBeforeReject = calls.length;
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
|
||||
// Nothing is written after the caller has seen the error.
|
||||
expect(calls.length).toBe(sentBeforeReject);
|
||||
expect(inFlight).toBe(0);
|
||||
});
|
||||
|
||||
it('retries the unprocessed deletes of the TTL sweep', async () => {
|
||||
const expired = [{ pk: { S: 'old' } }];
|
||||
let batchWrites = 0;
|
||||
handlers = {
|
||||
CreateTable: () => ({
|
||||
status: 400,
|
||||
__type: 'com.amazonaws.dynamodb.v20120810#ResourceInUseException',
|
||||
message: 'Table already exists',
|
||||
}),
|
||||
Scan: () => ({ Items: expired }),
|
||||
BatchWriteItem: (body) => {
|
||||
batchWrites += 1;
|
||||
return batchWrites === 1
|
||||
? { UnprocessedItems: body.RequestItems }
|
||||
: { UnprocessedItems: {} };
|
||||
},
|
||||
};
|
||||
|
||||
await stubClient({ bootstrapTables: true }).createTableIfNotExists(
|
||||
{
|
||||
TableName: TABLE,
|
||||
KeySchema: [{ AttributeName: 'pk', KeyType: 'HASH' }],
|
||||
AttributeDefinitions: [
|
||||
{ AttributeName: 'pk', AttributeType: 'S' },
|
||||
],
|
||||
BillingMode: 'PAY_PER_REQUEST',
|
||||
},
|
||||
'ttl',
|
||||
);
|
||||
|
||||
expect(calls).toEqual([
|
||||
'CreateTable',
|
||||
'Scan',
|
||||
'BatchWriteItem',
|
||||
'BatchWriteItem',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// Throttles are a service-side signal the emulator never produces, so these
|
||||
// drive the client against a stub answering in DynamoDB's error shape.
|
||||
describe('DDBClient — throttle logging', () => {
|
||||
let server: Server;
|
||||
let endpoint: string;
|
||||
let responses: { status: number; body: unknown }[];
|
||||
let warn: MockInstance<typeof console.warn>;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = createServer((req, res) => {
|
||||
req.resume();
|
||||
req.on('end', () => {
|
||||
const next = responses.shift() ?? { status: 200, body: {} };
|
||||
res.writeHead(next.status, {
|
||||
'content-type': 'application/x-amz-json-1.0',
|
||||
});
|
||||
res.end(JSON.stringify(next.body));
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) =>
|
||||
server.listen(0, '127.0.0.1', resolve),
|
||||
);
|
||||
endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const stubClient = () =>
|
||||
new DDBClient({
|
||||
port: 0,
|
||||
extensions: [],
|
||||
dynamo: {
|
||||
aws: {
|
||||
access_key: 'a',
|
||||
secret_key: 'b',
|
||||
region: 'eu-central-1',
|
||||
},
|
||||
endpoint,
|
||||
},
|
||||
} as unknown as IConfig);
|
||||
|
||||
const throttled = (reason = 'TableWriteKeyRangeThroughputExceeded') => ({
|
||||
status: 400,
|
||||
body: {
|
||||
__type: 'com.amazonaws.dynamodb.v20120810#ProvisionedThroughputExceededException',
|
||||
message:
|
||||
'The level of configured provisioned throughput for the table was exceeded.',
|
||||
ThrottlingReasons: [
|
||||
{
|
||||
reason,
|
||||
resource: `arn:aws:dynamodb:eu-central-1:0:table/${TABLE}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const ok = (body: unknown = {}) => ({ status: 200, body });
|
||||
|
||||
const ddbWarnings = () =>
|
||||
warn.mock.calls
|
||||
.map(([line]) => String(line))
|
||||
.filter((line) => line.startsWith('[ddb] '));
|
||||
|
||||
it('logs the key of a throttled update that succeeds on retry', async () => {
|
||||
responses = [
|
||||
throttled(),
|
||||
ok({
|
||||
Attributes: {
|
||||
pk: { S: 'hot' },
|
||||
sk: { S: 'counter' },
|
||||
v: { N: '1' },
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await stubClient().update(
|
||||
TABLE,
|
||||
{ pk: 'hot', sk: 'counter' },
|
||||
'SET v = :v',
|
||||
{ ':v': 1 },
|
||||
);
|
||||
|
||||
expect(result.Attributes?.v).toBe(1);
|
||||
expect(ddbWarnings()).toEqual([
|
||||
'[ddb] throttled UpdateItem kv-items pk=hot sk=counter in eu-central-1: TableWriteKeyRangeThroughputExceeded',
|
||||
]);
|
||||
}, 15_000);
|
||||
|
||||
it('logs only the key attributes of a throttled put', async () => {
|
||||
const client = stubClient();
|
||||
// On a managed deployment this creates nothing but learns the key schema.
|
||||
await client.createTableIfNotExists(tableSchema);
|
||||
responses = [throttled(), ok()];
|
||||
|
||||
await client.put(TABLE, {
|
||||
pk: 'user-1',
|
||||
sk: 'settings',
|
||||
secret: 'do-not-log',
|
||||
});
|
||||
|
||||
expect(ddbWarnings()).toEqual([
|
||||
'[ddb] throttled PutItem kv-items pk=user-1 sk=settings in eu-central-1: TableWriteKeyRangeThroughputExceeded',
|
||||
]);
|
||||
expect(warn.mock.calls.flat().join('\n')).not.toContain('do-not-log');
|
||||
}, 15_000);
|
||||
|
||||
it('logs a hot key once a minute and counts the throttles in between', async () => {
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000);
|
||||
const client = stubClient();
|
||||
const read = 'TableReadKeyRangeThroughputExceeded';
|
||||
|
||||
responses = [throttled(read), ok(), throttled(read), ok()];
|
||||
await client.get(TABLE, { pk: 'burst', sk: 'a' });
|
||||
await client.get(TABLE, { pk: 'burst', sk: 'a' });
|
||||
|
||||
now.mockReturnValue(1_000_000 + 61_000);
|
||||
responses = [throttled(read), ok()];
|
||||
await client.get(TABLE, { pk: 'burst', sk: 'a' });
|
||||
|
||||
expect(ddbWarnings()).toEqual([
|
||||
`[ddb] throttled GetItem kv-items pk=burst sk=a in eu-central-1: ${read}`,
|
||||
`[ddb] throttled GetItem kv-items pk=burst sk=a in eu-central-1: ${read} (+1 since last logged)`,
|
||||
]);
|
||||
}, 15_000);
|
||||
|
||||
it('names the partition a throttled query read', async () => {
|
||||
responses = [throttled('IndexReadKeyRangeThroughputExceeded'), ok()];
|
||||
|
||||
await stubClient().query(TABLE, { kind: 'note' }, 0, undefined, GSI);
|
||||
|
||||
expect(ddbWarnings()).toEqual([
|
||||
'[ddb] throttled Query kv-items index=by-kind kind=note in eu-central-1: IndexReadKeyRangeThroughputExceeded',
|
||||
]);
|
||||
}, 15_000);
|
||||
|
||||
it('names up to five keys of a throttled batch', async () => {
|
||||
responses = [
|
||||
throttled('TableReadKeyRangeThroughputExceeded'),
|
||||
ok({ Responses: {}, UnprocessedKeys: {} }),
|
||||
];
|
||||
|
||||
await stubClient().batchGet(
|
||||
Array.from({ length: 7 }, (_, index) => ({
|
||||
table: TABLE,
|
||||
items: { pk: `batch-${index}`, sk: 's' },
|
||||
})),
|
||||
);
|
||||
|
||||
expect(ddbWarnings()).toEqual([
|
||||
'[ddb] throttled BatchGetItem kv-items pk=batch-0 sk=s, pk=batch-1 sk=s, pk=batch-2 sk=s, pk=batch-3 sk=s, pk=batch-4 sk=s (+2 more) in eu-central-1: TableReadKeyRangeThroughputExceeded',
|
||||
]);
|
||||
}, 15_000);
|
||||
|
||||
it('logs batch items DynamoDB hands back unprocessed', async () => {
|
||||
const client = stubClient();
|
||||
await client.createTableIfNotExists(tableSchema);
|
||||
responses = [
|
||||
ok({
|
||||
UnprocessedItems: {
|
||||
[TABLE]: [
|
||||
{
|
||||
PutRequest: {
|
||||
Item: {
|
||||
pk: { S: 'left' },
|
||||
sk: { S: 'over' },
|
||||
v: { N: '1' },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
ok({ UnprocessedItems: {} }),
|
||||
];
|
||||
|
||||
await client.batchPut([
|
||||
{ table: TABLE, item: { pk: 'left', sk: 'over', v: 1 } },
|
||||
{ table: TABLE, item: { pk: 'done', sk: 'fine', v: 2 } },
|
||||
]);
|
||||
|
||||
expect(ddbWarnings()).toEqual([
|
||||
'[ddb] unprocessed BatchWriteItem kv-items pk=left sk=over in eu-central-1',
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not log errors that are not throttles', async () => {
|
||||
responses = [
|
||||
{
|
||||
status: 400,
|
||||
body: {
|
||||
__type: 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException',
|
||||
message: 'The conditional request failed',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
await expect(
|
||||
stubClient().del(
|
||||
TABLE,
|
||||
{ pk: 'guarded', sk: 'a' },
|
||||
{ condition: { expression: 'attribute_exists(pk)' } },
|
||||
),
|
||||
).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' });
|
||||
expect(ddbWarnings()).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -43,12 +43,6 @@ import { once } from 'node:events';
|
||||
import { Agent as httpsAgent } from 'node:https';
|
||||
import { HttpError } from '../../core/http';
|
||||
import type { IConfig, IDynamoConfig } from '../../types';
|
||||
import {
|
||||
awsCredentialsFrom,
|
||||
type AwsCredentials,
|
||||
} from '../../util/awsCredentials.js';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
import { runWithConcurrencyLimitSettled } from '../../util/concurrency.js';
|
||||
import { Span } from '../../util/span.js';
|
||||
import { PuterClient } from '../types';
|
||||
import {
|
||||
@@ -56,15 +50,14 @@ import {
|
||||
isRepairableMarshallError,
|
||||
repairForMarshall,
|
||||
} from './marshallRepair.js';
|
||||
import { attachThrottleLogging } from './throttleLog.js';
|
||||
|
||||
const LOCAL_DYNAMO_MEMORY_PREFIX = ':memory:';
|
||||
const localDynaliteEndpointPromises = new Map<string, Promise<string>>();
|
||||
const MAX_BATCH_WRITE_ITEMS = 25;
|
||||
// Chunks of one batch write in flight at once.
|
||||
const BATCH_WRITE_CONCURRENCY = 4;
|
||||
const MAX_BATCH_RETRIES = 8;
|
||||
const BATCH_RETRY_BASE_MS = 25;
|
||||
const MAX_BATCH_WRITE_RETRIES = 8;
|
||||
const BATCH_WRITE_RETRY_BASE_MS = 25;
|
||||
const MAX_BATCH_GET_RETRIES = 8;
|
||||
const BATCH_GET_RETRY_BASE_MS = 25;
|
||||
|
||||
// In-memory mode gives each DDBClient its own unique key, which keys
|
||||
// into a fresh dynalite server. This is what test parallelism needs:
|
||||
@@ -124,45 +117,21 @@ const chunkValues = <T>(values: T[], size: number): T[][] => {
|
||||
return chunks;
|
||||
};
|
||||
|
||||
type BatchWriteItems = NonNullable<BatchWriteCommandInput['RequestItems']>;
|
||||
type WriteRequest = BatchWriteItems[string][number];
|
||||
|
||||
/** Consumed capacity summed per table across the requests of one batch. */
|
||||
class ConsumedCapacityTally {
|
||||
#byTable = new Map<string, number>();
|
||||
|
||||
add(entries: { TableName?: string; CapacityUnits?: number }[] = []) {
|
||||
for (const { TableName, CapacityUnits } of entries) {
|
||||
if (!TableName) continue;
|
||||
this.#byTable.set(
|
||||
TableName,
|
||||
(this.#byTable.get(TableName) ?? 0) +
|
||||
Number(CapacityUnits ?? 0),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
entries() {
|
||||
return Array.from(this.#byTable, ([TableName, CapacityUnits]) => ({
|
||||
TableName,
|
||||
CapacityUnits,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = async (ms: number) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
};
|
||||
|
||||
// One caller writing an out-of-range number usually keeps doing it, so the
|
||||
// warning is per-target and throttled rather than one line per write.
|
||||
const repairWarnings = new BoundedTtlMap<string, true>({
|
||||
maxEntries: 1_000,
|
||||
ttlMs: 60_000,
|
||||
});
|
||||
const REPAIR_WARNING_INTERVAL_MS = 60_000;
|
||||
const lastRepairWarningAt = new Map<string, number>();
|
||||
|
||||
const warnRepaired = (target: string, message: string): void => {
|
||||
if (!repairWarnings.shouldEmit(target)) return;
|
||||
const now = Date.now();
|
||||
const lastWarnedAt = lastRepairWarningAt.get(target) ?? 0;
|
||||
if (now - lastWarnedAt < REPAIR_WARNING_INTERVAL_MS) return;
|
||||
|
||||
lastRepairWarningAt.set(target, now);
|
||||
console.warn(
|
||||
`[ddb] clamped an out-of-range value in ${target}: ${message}`,
|
||||
);
|
||||
@@ -203,7 +172,6 @@ export class DDBClient extends PuterClient {
|
||||
// key per instance so parallel clients don't share state, but
|
||||
// `recreateClient()` reuses this same key and so reuses the server.
|
||||
#localPathKey: string;
|
||||
#keyNamesByTable = new Map<string, string[]>();
|
||||
|
||||
constructor(config: IConfig) {
|
||||
super(config);
|
||||
@@ -252,19 +220,7 @@ export class DDBClient extends PuterClient {
|
||||
}
|
||||
|
||||
@Span('ddb.put', (table: string) => ({ 'db.table': table }))
|
||||
async put<T extends Record<string, unknown>>(
|
||||
table: string,
|
||||
item: T,
|
||||
opts?: {
|
||||
/** Refused (`ConditionalCheckFailedException`) unless it holds. */
|
||||
condition?: {
|
||||
expression: string;
|
||||
names?: Record<string, string>;
|
||||
values?: Record<string, unknown>;
|
||||
};
|
||||
},
|
||||
) {
|
||||
const condition = opts?.condition;
|
||||
async put<T extends Record<string, unknown>>(table: string, item: T) {
|
||||
const client = await this.#getDocumentClient();
|
||||
return sendRepairingValues(
|
||||
item,
|
||||
@@ -274,23 +230,6 @@ export class DDBClient extends PuterClient {
|
||||
TableName: table,
|
||||
Item: itemToWrite,
|
||||
ReturnConsumedCapacity: 'TOTAL',
|
||||
...(condition
|
||||
? {
|
||||
ConditionExpression: condition.expression,
|
||||
...(condition.names
|
||||
? {
|
||||
ExpressionAttributeNames:
|
||||
condition.names,
|
||||
}
|
||||
: {}),
|
||||
...(condition.values
|
||||
? {
|
||||
ExpressionAttributeValues:
|
||||
condition.values,
|
||||
}
|
||||
: {}),
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
),
|
||||
() => `a put to ${table}`,
|
||||
@@ -316,7 +255,9 @@ export class DDBClient extends PuterClient {
|
||||
{} as Record<string, Record<string, unknown>[]>,
|
||||
);
|
||||
|
||||
const requestItems = Object.entries(allRequestItemsPerTable).reduce(
|
||||
let requestItems: BatchGetCommandInput['RequestItems'] = Object.entries(
|
||||
allRequestItemsPerTable,
|
||||
).reduce(
|
||||
(acc, [table, keyList]) => {
|
||||
acc[table] = {
|
||||
Keys: keyList,
|
||||
@@ -329,30 +270,53 @@ export class DDBClient extends PuterClient {
|
||||
|
||||
const client = await this.#getDocumentClient();
|
||||
const responsesByTable = new Map<string, Record<string, unknown>[]>();
|
||||
const capacity = new ConsumedCapacityTally();
|
||||
const consumedCapacityByTable = new Map<string, number>();
|
||||
|
||||
const drained = await this.#sendWithUnprocessedRetry(
|
||||
requestItems,
|
||||
async (pending) => {
|
||||
const response = await client.send(
|
||||
new BatchGetCommand({
|
||||
RequestItems: pending,
|
||||
ReturnConsumedCapacity: 'TOTAL',
|
||||
}),
|
||||
for (let attempt = 0; attempt <= MAX_BATCH_GET_RETRIES; attempt++) {
|
||||
if (Object.keys(requestItems).length === 0) break;
|
||||
|
||||
const response = await client.send(
|
||||
new BatchGetCommand({
|
||||
RequestItems: requestItems,
|
||||
ReturnConsumedCapacity: 'TOTAL',
|
||||
}),
|
||||
);
|
||||
|
||||
for (const [table, items] of Object.entries(
|
||||
response.Responses ?? {},
|
||||
)) {
|
||||
const existing = responsesByTable.get(table) ?? [];
|
||||
existing.push(...items);
|
||||
responsesByTable.set(table, existing);
|
||||
}
|
||||
for (const entry of response.ConsumedCapacity ?? []) {
|
||||
if (!entry.TableName) continue;
|
||||
consumedCapacityByTable.set(
|
||||
entry.TableName,
|
||||
(consumedCapacityByTable.get(entry.TableName) ?? 0) +
|
||||
Number(entry.CapacityUnits ?? 0),
|
||||
);
|
||||
for (const [table, items] of Object.entries(
|
||||
response.Responses ?? {},
|
||||
)) {
|
||||
const existing = responsesByTable.get(table) ?? [];
|
||||
existing.push(...items);
|
||||
responsesByTable.set(table, existing);
|
||||
}
|
||||
capacity.add(response.ConsumedCapacity);
|
||||
return response.UnprocessedKeys as typeof pending | undefined;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (!drained) {
|
||||
const unprocessedKeys = response.UnprocessedKeys ?? {};
|
||||
if (Object.keys(unprocessedKeys).length === 0) {
|
||||
requestItems = {};
|
||||
break;
|
||||
}
|
||||
|
||||
requestItems = unprocessedKeys as NonNullable<
|
||||
BatchGetCommandInput['RequestItems']
|
||||
>;
|
||||
if (attempt < MAX_BATCH_GET_RETRIES) {
|
||||
const delayMs = Math.min(
|
||||
1000,
|
||||
BATCH_GET_RETRY_BASE_MS * 2 ** attempt,
|
||||
);
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(requestItems).length > 0) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Failed to batch get all items from DynamoDB',
|
||||
@@ -362,7 +326,12 @@ export class DDBClient extends PuterClient {
|
||||
|
||||
return {
|
||||
Responses: Object.fromEntries(responsesByTable),
|
||||
ConsumedCapacity: capacity.entries(),
|
||||
ConsumedCapacity: Array.from(consumedCapacityByTable.entries()).map(
|
||||
([TableName, CapacityUnits]) => ({
|
||||
TableName,
|
||||
CapacityUnits,
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -390,110 +359,129 @@ export class DDBClient extends PuterClient {
|
||||
);
|
||||
}
|
||||
|
||||
// Shared BatchWriteItem plumbing for batchPut/batchDel and the TTL sweep:
|
||||
// 25-item chunks a few at a time, UnprocessedItems retried, consumed
|
||||
// capacity summed per table across every request.
|
||||
async #batchWrite(params: { table: string; request: WriteRequest }[]) {
|
||||
// Shared BatchWriteItem plumbing for batchPut/batchDel: 25-item chunks,
|
||||
// UnprocessedItems retried with capped exponential backoff, consumed
|
||||
// capacity accumulated per table across every request.
|
||||
async #batchWrite(
|
||||
params: {
|
||||
table: string;
|
||||
request: NonNullable<
|
||||
BatchWriteCommandInput['RequestItems']
|
||||
>[string][number];
|
||||
}[],
|
||||
) {
|
||||
const consumedCapacityByTable = new Map<string, number>();
|
||||
if (params.length === 0) {
|
||||
return { ConsumedCapacity: [] };
|
||||
}
|
||||
|
||||
const accumulateConsumedCapacity = (
|
||||
consumedCapacityEntries:
|
||||
| Array<{ TableName?: string; CapacityUnits?: number }>
|
||||
| undefined,
|
||||
) => {
|
||||
if (!consumedCapacityEntries) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const consumedCapacityEntry of consumedCapacityEntries) {
|
||||
const table = consumedCapacityEntry.TableName;
|
||||
if (!table) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const existingUsage = consumedCapacityByTable.get(table) ?? 0;
|
||||
consumedCapacityByTable.set(
|
||||
table,
|
||||
existingUsage +
|
||||
Number(consumedCapacityEntry.CapacityUnits ?? 0),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const client = await this.#getDocumentClient();
|
||||
const capacity = new ConsumedCapacityTally();
|
||||
// Once a chunk fails, the rest aren't started. Those in flight settle
|
||||
// before the error is thrown, so no write lands after the caller's
|
||||
// cleanup.
|
||||
let failed = false;
|
||||
const chunks = chunkValues(params, MAX_BATCH_WRITE_ITEMS);
|
||||
|
||||
const outcomes = await runWithConcurrencyLimitSettled(
|
||||
chunkValues(params, MAX_BATCH_WRITE_ITEMS),
|
||||
BATCH_WRITE_CONCURRENCY,
|
||||
async (chunk) => {
|
||||
if (failed) return;
|
||||
const requestItems: BatchWriteItems = {};
|
||||
for (const { table, request } of chunk) {
|
||||
(requestItems[table] ??= []).push(request);
|
||||
}
|
||||
try {
|
||||
const drained = await this.#sendWithUnprocessedRetry(
|
||||
requestItems,
|
||||
async (pending) => {
|
||||
const response = await sendRepairingValues(
|
||||
pending,
|
||||
(itemsToWrite) =>
|
||||
client.send(
|
||||
new BatchWriteCommand({
|
||||
RequestItems: itemsToWrite,
|
||||
ReturnConsumedCapacity: 'TOTAL',
|
||||
}),
|
||||
),
|
||||
() =>
|
||||
`a batch write to ${Object.keys(pending).join(', ')}`,
|
||||
);
|
||||
capacity.add(response.ConsumedCapacity);
|
||||
return response.UnprocessedItems as
|
||||
BatchWriteItems | undefined;
|
||||
},
|
||||
);
|
||||
if (!drained) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Failed to batch write all items to DynamoDB',
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
);
|
||||
const rejected = outcomes.find(
|
||||
(outcome): outcome is PromiseRejectedResult =>
|
||||
outcome.status === 'rejected',
|
||||
);
|
||||
if (rejected) throw rejected.reason;
|
||||
for (const chunk of chunks) {
|
||||
let requestItems = chunk.reduce(
|
||||
(acc, curr) => {
|
||||
const tableRequests = acc[curr.table] ?? [];
|
||||
tableRequests.push(curr.request);
|
||||
acc[curr.table] = tableRequests;
|
||||
return acc;
|
||||
},
|
||||
{} as NonNullable<BatchWriteCommandInput['RequestItems']>,
|
||||
);
|
||||
|
||||
return { ConsumedCapacity: capacity.entries() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a batch request and resend whatever comes back unprocessed, with
|
||||
* capped exponential backoff. `send` returns the unprocessed remainder.
|
||||
* Resolves false when items are still left after the last retry.
|
||||
*/
|
||||
async #sendWithUnprocessedRetry<TItems extends object>(
|
||||
requestItems: TItems,
|
||||
send: (pending: TItems) => Promise<TItems | undefined>,
|
||||
): Promise<boolean> {
|
||||
let pending: TItems | undefined = requestItems;
|
||||
for (let attempt = 0; attempt <= MAX_BATCH_RETRIES; attempt++) {
|
||||
if (!pending || Object.keys(pending).length === 0) return true;
|
||||
pending = await send(pending);
|
||||
if (
|
||||
attempt < MAX_BATCH_RETRIES &&
|
||||
pending &&
|
||||
Object.keys(pending).length > 0
|
||||
for (
|
||||
let attempt = 0;
|
||||
attempt <= MAX_BATCH_WRITE_RETRIES;
|
||||
attempt++
|
||||
) {
|
||||
await sleep(Math.min(1000, BATCH_RETRY_BASE_MS * 2 ** attempt));
|
||||
if (Object.keys(requestItems).length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const response = await sendRepairingValues(
|
||||
requestItems,
|
||||
(itemsToWrite) =>
|
||||
client.send(
|
||||
new BatchWriteCommand({
|
||||
RequestItems: itemsToWrite,
|
||||
ReturnConsumedCapacity: 'TOTAL',
|
||||
}),
|
||||
),
|
||||
() =>
|
||||
`a batch write to ${Object.keys(requestItems).join(', ')}`,
|
||||
);
|
||||
accumulateConsumedCapacity(
|
||||
response.ConsumedCapacity as
|
||||
| Array<{ TableName?: string; CapacityUnits?: number }>
|
||||
| undefined,
|
||||
);
|
||||
|
||||
const unprocessedItems = response.UnprocessedItems ?? {};
|
||||
if (Object.keys(unprocessedItems).length === 0) {
|
||||
requestItems = {};
|
||||
break;
|
||||
}
|
||||
|
||||
requestItems = unprocessedItems as NonNullable<
|
||||
BatchWriteCommandInput['RequestItems']
|
||||
>;
|
||||
if (attempt < MAX_BATCH_WRITE_RETRIES) {
|
||||
const delayMs = Math.min(
|
||||
1000,
|
||||
BATCH_WRITE_RETRY_BASE_MS * 2 ** attempt,
|
||||
);
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(requestItems).length > 0) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Failed to batch write all items to DynamoDB',
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
}
|
||||
return !pending || Object.keys(pending).length === 0;
|
||||
|
||||
return {
|
||||
ConsumedCapacity: Array.from(consumedCapacityByTable.entries()).map(
|
||||
([TableName, CapacityUnits]) => ({
|
||||
TableName,
|
||||
CapacityUnits,
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@Span('ddb.del', (table: string) => ({ 'db.table': table }))
|
||||
async del<T extends Record<string, unknown>>(
|
||||
table: string,
|
||||
key: T,
|
||||
opts?: {
|
||||
returnOld?: boolean;
|
||||
/** Refused (`ConditionalCheckFailedException`) unless it holds. */
|
||||
condition?: {
|
||||
expression: string;
|
||||
names?: Record<string, string>;
|
||||
values?: Record<string, unknown>;
|
||||
};
|
||||
},
|
||||
opts?: { returnOld?: boolean },
|
||||
) {
|
||||
const command = new DeleteCommand({
|
||||
TableName: table,
|
||||
@@ -502,20 +490,6 @@ export class DDBClient extends PuterClient {
|
||||
// ALL_OLD makes the delete an atomic claim: exactly one caller
|
||||
// gets the attributes back.
|
||||
...(opts?.returnOld ? { ReturnValues: 'ALL_OLD' as const } : {}),
|
||||
...(opts?.condition
|
||||
? {
|
||||
ConditionExpression: opts.condition.expression,
|
||||
...(opts.condition.names
|
||||
? { ExpressionAttributeNames: opts.condition.names }
|
||||
: {}),
|
||||
...(opts.condition.values
|
||||
? {
|
||||
ExpressionAttributeValues:
|
||||
opts.condition.values,
|
||||
}
|
||||
: {}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
const client = await this.#getDocumentClient();
|
||||
@@ -637,25 +611,10 @@ export class DDBClient extends PuterClient {
|
||||
);
|
||||
}
|
||||
|
||||
/** Lets throttle logs name a put's key without logging the item. */
|
||||
registerKeySchema(
|
||||
table: string,
|
||||
keySchema: NonNullable<CreateTableCommandInput['KeySchema']>,
|
||||
) {
|
||||
this.#keyNamesByTable.set(
|
||||
table,
|
||||
keySchema.map((key) => key.AttributeName!),
|
||||
);
|
||||
}
|
||||
|
||||
async createTableIfNotExists(
|
||||
params: CreateTableCommandInput,
|
||||
ttlAttribute?: string,
|
||||
) {
|
||||
if (params.TableName && params.KeySchema) {
|
||||
this.registerKeySchema(params.TableName, params.KeySchema);
|
||||
}
|
||||
|
||||
// Real-AWS deployments provision tables externally (Terraform / IaC),
|
||||
// so we no-op there by default. Self-hosters pointing at
|
||||
// dynamodb-local opt in via `dynamo.bootstrapTables: true`.
|
||||
@@ -721,17 +680,39 @@ export class DDBClient extends PuterClient {
|
||||
}
|
||||
|
||||
#bindAwsClient() {
|
||||
const credentials = awsCredentialsFrom(this.#ddbConfig.aws);
|
||||
if (!credentials) {
|
||||
const accessKeyId = this.#ddbConfig.aws?.access_key;
|
||||
const secretAccessKey = this.#ddbConfig.aws?.secret_key;
|
||||
|
||||
if (!accessKeyId || !secretAccessKey) {
|
||||
throw new Error(
|
||||
'DynamoDB aws config requires both `access_key` and `secret_key`, or `useCredentialChain`',
|
||||
'DynamoDB aws config requires both `access_key` and `secret_key`',
|
||||
);
|
||||
}
|
||||
|
||||
this.#bindClient({
|
||||
credentials,
|
||||
const ddbClient = new DynamoDBClient({
|
||||
credentials: {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
},
|
||||
maxAttempts: 3,
|
||||
requestHandler: new NodeHttpHandler({
|
||||
connectionTimeout: 5000,
|
||||
requestTimeout: 5000,
|
||||
httpsAgent: new httpsAgent({ keepAlive: true }),
|
||||
}),
|
||||
...(this.#ddbConfig.endpoint
|
||||
? { endpoint: this.#ddbConfig.endpoint }
|
||||
: {}),
|
||||
region: this.#ddbConfig.aws?.region || 'us-west-2',
|
||||
endpoint: this.#ddbConfig.endpoint,
|
||||
});
|
||||
|
||||
this.#documentClient = DynamoDBDocumentClient.from(ddbClient, {
|
||||
marshallOptions: {
|
||||
removeUndefinedValues: true,
|
||||
},
|
||||
unmarshallOptions: {
|
||||
wrapNumbers: clampStoredNumber,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -739,36 +720,21 @@ export class DDBClient extends PuterClient {
|
||||
const endpoint = await getOrCreateLocalDynaliteEndpoint(
|
||||
this.#localPathKey,
|
||||
);
|
||||
this.#bindClient({
|
||||
credentials: { accessKeyId: 'fake', secretAccessKey: 'fake' },
|
||||
region: 'us-west-2',
|
||||
endpoint,
|
||||
});
|
||||
}
|
||||
|
||||
#bindClient({
|
||||
credentials,
|
||||
region,
|
||||
endpoint,
|
||||
}: {
|
||||
credentials: AwsCredentials;
|
||||
region: string;
|
||||
endpoint?: string;
|
||||
}) {
|
||||
const ddbClient = new DynamoDBClient({
|
||||
credentials,
|
||||
credentials: {
|
||||
accessKeyId: 'fake',
|
||||
secretAccessKey: 'fake',
|
||||
},
|
||||
maxAttempts: 3,
|
||||
requestHandler: new NodeHttpHandler({
|
||||
connectionTimeout: 5000,
|
||||
requestTimeout: 5000,
|
||||
httpsAgent: new httpsAgent({ keepAlive: true }),
|
||||
}),
|
||||
...(endpoint ? { endpoint } : {}),
|
||||
region,
|
||||
endpoint,
|
||||
region: 'us-west-2',
|
||||
});
|
||||
attachThrottleLogging(ddbClient, region, (table) =>
|
||||
this.#keyNamesByTable.get(table),
|
||||
);
|
||||
|
||||
this.#documentClient = DynamoDBDocumentClient.from(ddbClient, {
|
||||
marshallOptions: {
|
||||
@@ -795,15 +761,14 @@ export class DDBClient extends PuterClient {
|
||||
const scan = await client.send(
|
||||
new ScanCommand({
|
||||
TableName: table,
|
||||
// 0 means no expiry, so it must never sweep.
|
||||
FilterExpression: '#ttl < :now AND #ttl <> :none',
|
||||
FilterExpression: '#ttl < :now',
|
||||
ExpressionAttributeNames: {
|
||||
'#ttl': ttlAttribute,
|
||||
...Object.fromEntries(
|
||||
keyNames.map((key) => [`#k_${key}`, key]),
|
||||
),
|
||||
},
|
||||
ExpressionAttributeValues: { ':now': now, ':none': 0 },
|
||||
ExpressionAttributeValues: { ':now': now },
|
||||
ProjectionExpression: keyNames
|
||||
.map((key) => `#k_${key}`)
|
||||
.join(', '),
|
||||
@@ -814,22 +779,27 @@ export class DDBClient extends PuterClient {
|
||||
);
|
||||
|
||||
lastEvaluatedKey = scan.LastEvaluatedKey as
|
||||
Record<string, unknown> | undefined;
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
const items = scan.Items;
|
||||
if (!items || items.length === 0) continue;
|
||||
|
||||
await this.#batchWrite(
|
||||
items.map((item) => ({
|
||||
table,
|
||||
request: {
|
||||
DeleteRequest: {
|
||||
Key: Object.fromEntries(
|
||||
keyNames.map((key) => [key, item[key]]),
|
||||
),
|
||||
const chunks = chunkValues(items, MAX_BATCH_WRITE_ITEMS);
|
||||
for (const chunk of chunks) {
|
||||
await client.send(
|
||||
new BatchWriteCommand({
|
||||
RequestItems: {
|
||||
[table]: chunk.map((item) => ({
|
||||
DeleteRequest: {
|
||||
Key: Object.fromEntries(
|
||||
keyNames.map((key) => [key, item[key]]),
|
||||
),
|
||||
},
|
||||
})),
|
||||
},
|
||||
},
|
||||
})),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
} while (lastEvaluatedKey);
|
||||
}
|
||||
}
|
||||
@@ -1,245 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
|
||||
const THROTTLE_ERROR_NAMES = new Set([
|
||||
'ProvisionedThroughputExceededException',
|
||||
'ThrottlingException',
|
||||
'RequestLimitExceeded',
|
||||
]);
|
||||
|
||||
// A hot key throttles in bursts, so each target logs at most once a minute and
|
||||
// the next line says how many were folded into it.
|
||||
const LOG_INTERVAL_MS = 60_000;
|
||||
const MAX_TRACKED_TARGETS = 1_000;
|
||||
const MAX_KEYS_PER_LINE = 5;
|
||||
const MAX_VALUE_LENGTH = 120;
|
||||
|
||||
type AnyRecord = Record<string, unknown>;
|
||||
type KeyNamesFor = (table: string) => string[] | undefined;
|
||||
interface Target {
|
||||
table: string;
|
||||
keys: string;
|
||||
}
|
||||
|
||||
// No TTL: an entry past its interval still carries the count to report.
|
||||
const lastLogged = new BoundedTtlMap<string, { at: number; folded: number }>({
|
||||
maxEntries: MAX_TRACKED_TARGETS,
|
||||
});
|
||||
|
||||
const logOnce = (target: string, detail?: string): void => {
|
||||
const now = Date.now();
|
||||
const previous = lastLogged.get(target);
|
||||
if (previous && now - previous.at < LOG_INTERVAL_MS) {
|
||||
previous.folded += 1;
|
||||
return;
|
||||
}
|
||||
lastLogged.set(target, { at: now, folded: 0 });
|
||||
|
||||
const folded = previous?.folded
|
||||
? ` (+${previous.folded} since last logged)`
|
||||
: '';
|
||||
console.warn(`[ddb] ${target}${detail ? `: ${detail}` : ''}${folded}`);
|
||||
};
|
||||
|
||||
// Inputs reach this middleware marshalled and outputs arrive unmarshalled, so
|
||||
// a key value can be either shape.
|
||||
const formatValue = (value: unknown): string => {
|
||||
let text: string;
|
||||
if (value && typeof value === 'object') {
|
||||
const attr = value as { S?: string; N?: string; B?: unknown };
|
||||
text =
|
||||
attr.S ??
|
||||
attr.N ??
|
||||
(attr.B !== undefined ? '<binary>' : JSON.stringify(value));
|
||||
} else {
|
||||
text = String(value);
|
||||
}
|
||||
return text.length > MAX_VALUE_LENGTH
|
||||
? `${text.slice(0, MAX_VALUE_LENGTH)}...`
|
||||
: text;
|
||||
};
|
||||
|
||||
const formatKey = (key: unknown, names?: string[]): string => {
|
||||
if (!key || typeof key !== 'object') return '<no key>';
|
||||
const record = key as AnyRecord;
|
||||
const parts = (names ?? Object.keys(record))
|
||||
.filter((name) => name in record)
|
||||
.map((name) => `${name}=${formatValue(record[name])}`);
|
||||
return parts.length > 0 ? parts.join(' ') : '<no key>';
|
||||
};
|
||||
|
||||
// A put carries the whole item; only its key attributes are logged, never values.
|
||||
const formatItemKey = (
|
||||
table: string,
|
||||
item: unknown,
|
||||
keyNamesFor: KeyNamesFor,
|
||||
): string => {
|
||||
const names = keyNamesFor(table);
|
||||
return names ? formatKey(item, names) : '<key schema unknown>';
|
||||
};
|
||||
|
||||
const formatList = (keys: string[]): string => {
|
||||
const shown = keys.slice(0, MAX_KEYS_PER_LINE).join(', ');
|
||||
const hidden = keys.length - MAX_KEYS_PER_LINE;
|
||||
return hidden > 0 ? `${shown} (+${hidden} more)` : shown;
|
||||
};
|
||||
|
||||
const formatQueryKey = (input: AnyRecord): string => {
|
||||
const values = (input.ExpressionAttributeValues ?? {}) as AnyRecord;
|
||||
const placeholders =
|
||||
String(input.KeyConditionExpression ?? '').match(/:[\w-]+/g) ?? [];
|
||||
const parts = placeholders.map(
|
||||
(placeholder) =>
|
||||
`${placeholder.slice(1)}=${formatValue(values[placeholder])}`,
|
||||
);
|
||||
const index = input.IndexName ? `index=${input.IndexName} ` : '';
|
||||
return index + (parts.length > 0 ? parts.join(' ') : '<no key>');
|
||||
};
|
||||
|
||||
const describeBatchWrite = (
|
||||
requestItems: unknown,
|
||||
keyNamesFor: KeyNamesFor,
|
||||
): Target[] =>
|
||||
Object.entries((requestItems ?? {}) as AnyRecord).map(
|
||||
([table, requests]) => ({
|
||||
table,
|
||||
keys: formatList(
|
||||
((requests ?? []) as AnyRecord[]).map((request) => {
|
||||
const put = request.PutRequest as AnyRecord | undefined;
|
||||
if (put) return formatItemKey(table, put.Item, keyNamesFor);
|
||||
const del = request.DeleteRequest as AnyRecord | undefined;
|
||||
return formatKey(del?.Key);
|
||||
}),
|
||||
),
|
||||
}),
|
||||
);
|
||||
|
||||
const describeBatchGet = (requestItems: unknown): Target[] =>
|
||||
Object.entries((requestItems ?? {}) as AnyRecord).map(([table, entry]) => ({
|
||||
table,
|
||||
keys: formatList(
|
||||
((entry as { Keys?: unknown[] })?.Keys ?? []).map((key) =>
|
||||
formatKey(key),
|
||||
),
|
||||
),
|
||||
}));
|
||||
|
||||
const describeRequest = (
|
||||
commandName: string,
|
||||
input: AnyRecord,
|
||||
keyNamesFor: KeyNamesFor,
|
||||
): Target[] => {
|
||||
const table = String(input.TableName ?? '');
|
||||
switch (commandName) {
|
||||
case 'GetItemCommand':
|
||||
case 'UpdateItemCommand':
|
||||
case 'DeleteItemCommand':
|
||||
return [{ table, keys: formatKey(input.Key) }];
|
||||
case 'PutItemCommand':
|
||||
return [
|
||||
{ table, keys: formatItemKey(table, input.Item, keyNamesFor) },
|
||||
];
|
||||
case 'QueryCommand':
|
||||
return [{ table, keys: formatQueryKey(input) }];
|
||||
case 'BatchWriteItemCommand':
|
||||
return describeBatchWrite(input.RequestItems, keyNamesFor);
|
||||
case 'BatchGetItemCommand':
|
||||
return describeBatchGet(input.RequestItems);
|
||||
default:
|
||||
return [{ table, keys: '<no key>' }];
|
||||
}
|
||||
};
|
||||
|
||||
const throttleReasons = (error: unknown): string => {
|
||||
const err = error as {
|
||||
name?: string;
|
||||
ThrottlingReasons?: { reason?: string }[];
|
||||
throttlingReasons?: { reason?: string }[];
|
||||
};
|
||||
const reasons = (err.ThrottlingReasons ?? err.throttlingReasons ?? [])
|
||||
.map((entry) => entry.reason)
|
||||
.filter(Boolean);
|
||||
return reasons.length > 0 ? reasons.join(', ') : String(err.name);
|
||||
};
|
||||
|
||||
/**
|
||||
* Log the table and key of every throttled attempt, including the ones the SDK
|
||||
* retries through, plus batch items DynamoDB hands back unprocessed. Sits
|
||||
* inside the retry loop because the CloudWatch throttle metrics count attempts,
|
||||
* and most throttled attempts succeed on retry.
|
||||
*/
|
||||
export const attachThrottleLogging = (
|
||||
client: DynamoDBClient,
|
||||
region: string,
|
||||
keyNamesFor: KeyNamesFor,
|
||||
): void => {
|
||||
client.middlewareStack.addRelativeTo(
|
||||
<TArgs extends { input: unknown }, TResult extends { output: unknown }>(
|
||||
next: (args: TArgs) => Promise<TResult>,
|
||||
context: { commandName?: string },
|
||||
) =>
|
||||
async (args: TArgs): Promise<TResult> => {
|
||||
const commandName = context.commandName ?? '';
|
||||
const operation = commandName.replace(/Command$/, '');
|
||||
try {
|
||||
const result = await next(args);
|
||||
const output = result.output as {
|
||||
UnprocessedItems?: unknown;
|
||||
UnprocessedKeys?: unknown;
|
||||
};
|
||||
const leftovers = [
|
||||
...describeBatchWrite(
|
||||
output?.UnprocessedItems,
|
||||
keyNamesFor,
|
||||
),
|
||||
...describeBatchGet(output?.UnprocessedKeys),
|
||||
];
|
||||
for (const { table, keys } of leftovers) {
|
||||
logOnce(
|
||||
`unprocessed ${operation} ${table} ${keys} in ${region}`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (THROTTLE_ERROR_NAMES.has((error as Error)?.name)) {
|
||||
const targets = describeRequest(
|
||||
commandName,
|
||||
args.input as AnyRecord,
|
||||
keyNamesFor,
|
||||
);
|
||||
for (const { table, keys } of targets) {
|
||||
logOnce(
|
||||
`throttled ${operation} ${table} ${keys} in ${region}`,
|
||||
throttleReasons(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'throttleKeyLogger',
|
||||
relation: 'after',
|
||||
toMiddleware: 'retryMiddleware',
|
||||
},
|
||||
);
|
||||
};
|
||||
@@ -425,3 +425,86 @@ describe('EmailClient — share notification templates', () => {
|
||||
expect(html).toContain('href="https://puter.test"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('EmailClient.clean', () => {
|
||||
const clean = (email: string) => startClient().clean(email);
|
||||
|
||||
it('strips subaddressing on ordinary domains', () => {
|
||||
expect(clean('[email protected]')).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('ignores dots and subaddressing for gmail', () => {
|
||||
expect(clean('[email protected]')).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('canonicalizes googlemail onto gmail', () => {
|
||||
expect(clean('[email protected]')).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('keeps yahoo subaddressing, which yahoo treats as distinct', () => {
|
||||
expect(clean('[email protected]')).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('ignores dots for icloud aliases', () => {
|
||||
expect(clean('[email protected]')).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('returns anything that is not an address unchanged', () => {
|
||||
expect(clean('not-an-email')).toBe('not-an-email');
|
||||
expect(clean('@nolocal.test')).toBe('@nolocal.test');
|
||||
});
|
||||
});
|
||||
|
||||
describe('EmailClient.validate', () => {
|
||||
it('waves everything through in a dev environment', async () => {
|
||||
const client = startClient({ env: 'dev' } as Partial<IConfig>);
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a blocked domain suffix', async () => {
|
||||
const client = startClient({
|
||||
blockedEmailDomains: ['blocked.test'],
|
||||
} as unknown as Partial<IConfig>);
|
||||
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(
|
||||
false,
|
||||
);
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('matches the blocklist against the cleaned address', async () => {
|
||||
const client = startClient({
|
||||
blockedEmailDomains: ['@gmail.com'],
|
||||
} as unknown as Partial<IConfig>);
|
||||
await expect(
|
||||
client.validate('[email protected]'),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('lets a registered validator veto an address', async () => {
|
||||
const client = startClient();
|
||||
const seen: string[] = [];
|
||||
client.addValidator((email) => {
|
||||
seen.push(email);
|
||||
return !email.startsWith('spam');
|
||||
});
|
||||
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(
|
||||
false,
|
||||
);
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(true);
|
||||
// Validators always see the canonical form.
|
||||
expect(seen).toEqual(['[email protected]', '[email protected]']);
|
||||
});
|
||||
|
||||
it('supports asynchronous validators', async () => {
|
||||
const client = startClient();
|
||||
client.addValidator(async (email) => email !== '[email protected]');
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(false);
|
||||
await expect(client.validate('[email protected]')).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -71,12 +71,54 @@ export interface SendMailOptions {
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export type EmailValidator = (email: string) => Promise<boolean> | boolean;
|
||||
|
||||
interface CompiledTemplate {
|
||||
subject: ReturnType<typeof template>;
|
||||
html: ReturnType<typeof template>;
|
||||
text?: ReturnType<typeof template>;
|
||||
}
|
||||
|
||||
// -- Clean-email rules ------------------------------------------------
|
||||
|
||||
type CleanRule = (parts: { local: string; domain: string }) => {
|
||||
local: string;
|
||||
domain: string;
|
||||
};
|
||||
|
||||
const CLEAN_RULES: Record<string, CleanRule> = {
|
||||
dots_dont_matter: ({ local, domain }) => ({
|
||||
local: local.replace(/\./g, ''),
|
||||
domain,
|
||||
}),
|
||||
remove_subaddressing: ({ local, domain }) => ({
|
||||
local: local.split('+')[0],
|
||||
domain,
|
||||
}),
|
||||
};
|
||||
|
||||
const PROVIDER_RULES: Record<string, { apply: string[]; skip: string[] }> = {
|
||||
gmail: { apply: ['dots_dont_matter'], skip: [] },
|
||||
icloud: { apply: ['dots_dont_matter'], skip: [] },
|
||||
yahoo: { apply: [], skip: ['remove_subaddressing'] },
|
||||
};
|
||||
|
||||
const DOMAIN_TO_PROVIDER: Record<string, string> = {
|
||||
'gmail.com': 'gmail',
|
||||
'googlemail.com': 'gmail',
|
||||
'yahoo.com': 'yahoo',
|
||||
'yahoo.co.uk': 'yahoo',
|
||||
'yahoo.ca': 'yahoo',
|
||||
'yahoo.com.au': 'yahoo',
|
||||
'icloud.com': 'icloud',
|
||||
'me.com': 'icloud',
|
||||
'mac.com': 'icloud',
|
||||
};
|
||||
|
||||
const DOMAIN_ALIASES: Record<string, string> = {
|
||||
'googlemail.com': 'gmail.com',
|
||||
};
|
||||
|
||||
// -- EmailClient ------------------------------------------------------
|
||||
|
||||
/**
|
||||
@@ -84,8 +126,8 @@ interface CompiledTemplate {
|
||||
*
|
||||
* - Template-based outbound mail (via `send`)
|
||||
* - Raw nodemailer passthrough (via `sendRaw`)
|
||||
*
|
||||
* Address normalization and the domain blocklist live in `util/email.ts`.
|
||||
* - Canonical-form normalization for dedup (via `clean`)
|
||||
* - Policy + extensible validation (via `validate`)
|
||||
*/
|
||||
export class EmailClient extends PuterClient {
|
||||
private transport: ReturnType<typeof nodemailer.createTransport> | null =
|
||||
@@ -93,6 +135,7 @@ export class EmailClient extends PuterClient {
|
||||
private compiledTemplates: Partial<
|
||||
Record<EmailTemplateName, CompiledTemplate>
|
||||
> = {};
|
||||
private validators: EmailValidator[] = [];
|
||||
|
||||
constructor(config: IConfig) {
|
||||
super(config);
|
||||
@@ -175,6 +218,72 @@ export class EmailClient extends PuterClient {
|
||||
});
|
||||
}
|
||||
|
||||
// -- Public API: clean / validate ---------------------------------
|
||||
|
||||
/**
|
||||
* Normalize an email to its canonical form for dedup comparisons. Applies
|
||||
* provider-specific rules (e.g. Gmail ignores dots in the local part) plus
|
||||
* generic subaddressing removal.
|
||||
*/
|
||||
clean(email: string): string {
|
||||
let [local, domain] = email.split('@');
|
||||
if (!local || !domain) return email;
|
||||
|
||||
if (DOMAIN_ALIASES[domain]) {
|
||||
domain = DOMAIN_ALIASES[domain];
|
||||
}
|
||||
|
||||
// Default: strip subaddressing on everything unless provider skips it
|
||||
const ruleNames = new Set<string>(['remove_subaddressing']);
|
||||
const provider = DOMAIN_TO_PROVIDER[domain];
|
||||
const rules = provider ? PROVIDER_RULES[provider] : undefined;
|
||||
|
||||
if (rules) {
|
||||
rules.apply.forEach((r) => ruleNames.add(r));
|
||||
rules.skip.forEach((r) => ruleNames.delete(r));
|
||||
}
|
||||
|
||||
let parts = { local, domain };
|
||||
for (const name of ruleNames) {
|
||||
parts = CLEAN_RULES[name](parts);
|
||||
}
|
||||
|
||||
return `${parts.local}@${parts.domain}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether an email is allowed to be used. Checks domain blocklist
|
||||
* plus any registered validators (services can call `addValidator()` to
|
||||
* register custom policy hooks).
|
||||
*/
|
||||
async validate(email: string): Promise<boolean> {
|
||||
if (this.config.env === 'dev') return true;
|
||||
|
||||
const cleaned = this.clean(email);
|
||||
|
||||
const blocked = this.config.blockedEmailDomains;
|
||||
if (Array.isArray(blocked)) {
|
||||
for (const suffix of blocked) {
|
||||
if (cleaned.endsWith(suffix)) return false;
|
||||
}
|
||||
}
|
||||
|
||||
for (const validator of this.validators) {
|
||||
const ok = await validator(cleaned);
|
||||
if (!ok) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a custom validation hook. Services can call this during their
|
||||
* startup to veto specific emails (e.g. a disposable-email service).
|
||||
*/
|
||||
addValidator(fn: EmailValidator): void {
|
||||
this.validators.push(fn);
|
||||
}
|
||||
|
||||
// -- Internals ----------------------------------------------------
|
||||
|
||||
private defaultFrom(): string {
|
||||
|
||||
@@ -352,10 +352,10 @@ [email protected] immediately.
|
||||
html: `
|
||||
<p>Hi there,</p>
|
||||
<p>{{team_name}} has created a Puter account for you.</p>
|
||||
<p>Username: <b>{{username}}</b></p>
|
||||
<p>{{team_name}} will give you a temporary password. You will be asked to choose
|
||||
your own the first time you sign in. The temporary one stops working then, and it
|
||||
expires on its own if unused.</p>
|
||||
<p>Username: <b>{{username}}</b><br>
|
||||
Temporary password: <b>{{temporary_password}}</b></p>
|
||||
<p>You will be asked to choose your own password the first time you sign in.
|
||||
This temporary one stops working then, and it expires on its own if unused.</p>
|
||||
<p>What this means:</p>
|
||||
<ul>
|
||||
<li>This account belongs to {{team_name}}. They pay for it and can close it.</li>
|
||||
@@ -364,21 +364,6 @@ expires on its own if unused.</p>
|
||||
as you. Choose your own password promptly.</li>
|
||||
</ul>
|
||||
<p>Sincerely,</p>
|
||||
<p>Puter</p>
|
||||
`,
|
||||
},
|
||||
team_account_reissued: {
|
||||
subject: 'A new temporary password for your {{team_name}} account',
|
||||
html: `
|
||||
<p>Hi there,</p>
|
||||
<p>{{team_name}} has issued a new temporary password for your Puter account
|
||||
<b>{{username}}</b>. Any temporary password you were given before this one has
|
||||
stopped working.</p>
|
||||
<p>{{team_name}} will give you the new one. You will be asked to choose your own
|
||||
password the first time you sign in; the temporary one stops working then, and
|
||||
it expires on its own if unused.</p>
|
||||
<p>If you were not expecting this, ask {{team_name}} -- only they can issue it.</p>
|
||||
<p>Sincerely,</p>
|
||||
<p>Puter</p>
|
||||
`,
|
||||
},
|
||||
@@ -419,22 +404,6 @@ will be asked to choose your own the next time you sign in.</p>
|
||||
<p>If you did not expect this, the record of who reset it and when is in your
|
||||
team activity, along with every sign-in to your account.</p>
|
||||
<p>Sincerely,</p>
|
||||
<p>Puter</p>
|
||||
`,
|
||||
},
|
||||
team_2fa_reset: {
|
||||
subject:
|
||||
'Two-factor authentication was reset on your {{team_name}} account',
|
||||
html: `
|
||||
<p>Hi there,</p>
|
||||
<p>An administrator of {{team_name}} reset two-factor authentication on your
|
||||
Puter account <b>{{username}}</b>, so you can set it up again from a device you
|
||||
still have. You have been signed out everywhere.</p>
|
||||
<p>Your password is unchanged. If your team requires two-factor authentication,
|
||||
you will be asked to enrol the next time you sign in.</p>
|
||||
<p>If you did not expect this, the record of who reset it and when is in your
|
||||
team activity, along with every sign-in to your account.</p>
|
||||
<p>Sincerely,</p>
|
||||
<p>Puter</p>
|
||||
`,
|
||||
},
|
||||
|
||||
@@ -1,9 +1,3 @@
|
||||
import { trace } from '@opentelemetry/api';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
InMemorySpanExporter,
|
||||
SimpleSpanProcessor,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
@@ -142,30 +136,6 @@ describe('EventClient', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('repeat emits', () => {
|
||||
it('reaches listeners added after the key was first emitted', () => {
|
||||
target.emit(`${key}.a.b`, {}, {});
|
||||
const wildcard = vi.fn();
|
||||
const exact = vi.fn();
|
||||
target.on(`${key}.a.*`, wildcard);
|
||||
target.on(`${key}.a.b`, exact);
|
||||
target.emit(`${key}.a.b`, {}, {});
|
||||
expect(wildcard).toHaveBeenCalledTimes(1);
|
||||
expect(exact).toHaveBeenCalledTimes(1);
|
||||
expect(target.hasListeners(`${key}.a.b`)).toBe(true);
|
||||
});
|
||||
|
||||
it('does not skip a listener when an earlier one unsubscribes mid-dispatch', () => {
|
||||
const second = vi.fn();
|
||||
const first = vi.fn(() => target.off(key, first));
|
||||
target.on(key, first);
|
||||
target.on(key, second);
|
||||
target.emit(key, {}, {});
|
||||
expect(first).toHaveBeenCalledTimes(1);
|
||||
expect(second).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('emitAndWait', () => {
|
||||
it('awaits async listeners before resolving', async () => {
|
||||
let resolved = false;
|
||||
@@ -213,122 +183,6 @@ describe('EventClient', () => {
|
||||
await target.emitAndWait(`${key}.child`, {}, {});
|
||||
expect(resolved).toBe(true);
|
||||
});
|
||||
|
||||
describe('tracing', () => {
|
||||
const exporter = new InMemorySpanExporter();
|
||||
const provider = new BasicTracerProvider({
|
||||
spanProcessors: [new SimpleSpanProcessor(exporter)],
|
||||
});
|
||||
const spansFor = (eventKey: string) =>
|
||||
exporter
|
||||
.getFinishedSpans()
|
||||
.filter(
|
||||
(s) =>
|
||||
s.name === 'event.emitAndWait' &&
|
||||
s.attributes['event.key'] === eventKey,
|
||||
);
|
||||
|
||||
beforeAll(() => {
|
||||
trace.setGlobalTracerProvider(provider);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await provider.shutdown();
|
||||
trace.disable();
|
||||
});
|
||||
|
||||
it('resolves without a span when nothing listens', async () => {
|
||||
const meta: { listener_failed?: boolean } = {};
|
||||
await expect(
|
||||
target.emitAndWait(`${key}.child`, {}, meta),
|
||||
).resolves.toBeUndefined();
|
||||
expect(meta).toEqual({});
|
||||
expect(spansFor(`${key}.child`)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('spans the wait whenever a listener matches, wildcards included', async () => {
|
||||
target.on(`${key}.*`, vi.fn());
|
||||
await expect(
|
||||
target.emitAndWait(`${key}.child`, {}, {}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(spansFor(`${key}.child`)).toHaveLength(1);
|
||||
|
||||
extensionStore.events[`${key}.ext`] = [vi.fn()];
|
||||
try {
|
||||
await target.emitAndWait(`${key}.ext`, {}, {});
|
||||
} finally {
|
||||
delete extensionStore.events[`${key}.ext`];
|
||||
}
|
||||
expect(spansFor(`${key}.ext`)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('emitAndWaitConcurrently', () => {
|
||||
it('runs every listener at once, wildcards included, and awaits them all', async () => {
|
||||
let inFlight = 0;
|
||||
let peak = 0;
|
||||
const contribute =
|
||||
(share: number) => async (_k: string, data: unknown) => {
|
||||
inFlight++;
|
||||
peak = Math.max(peak, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
inFlight--;
|
||||
(data as { total: number }).total += share;
|
||||
};
|
||||
target.on(`${key}.*`, contribute(1));
|
||||
target.on(`${key}.child`, contribute(10));
|
||||
target.on(`${key}.child`, contribute(100));
|
||||
|
||||
const data = { total: 0 };
|
||||
await target.emitAndWaitConcurrently(`${key}.child`, data, {});
|
||||
|
||||
expect(peak).toBe(3);
|
||||
expect(data.total).toBe(111);
|
||||
});
|
||||
|
||||
it('flags a failed exact-key listener and still awaits the rest', async () => {
|
||||
const errSpy = vi
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => {});
|
||||
let finished = false;
|
||||
target.on(key, () => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
target.on(key, async () => {
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
finished = true;
|
||||
});
|
||||
const meta: Record<string, unknown> = {};
|
||||
await target.emitAndWaitConcurrently(key, {}, meta);
|
||||
expect(finished).toBe(true);
|
||||
expect(meta.listener_failed).toBe(true);
|
||||
expect(errSpy).toHaveBeenCalledWith(
|
||||
'Error in event listener for event',
|
||||
key,
|
||||
expect.any(Error),
|
||||
);
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('leaves the flag alone when only a wildcard listener fails', async () => {
|
||||
const errSpy = vi
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => {});
|
||||
target.on(`${key}.*`, () => {
|
||||
throw new Error('observer');
|
||||
});
|
||||
const meta: Record<string, unknown> = {};
|
||||
await target.emitAndWaitConcurrently(`${key}.child`, {}, meta);
|
||||
expect(meta.listener_failed).toBeUndefined();
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('resolves at once with nobody listening', async () => {
|
||||
await expect(
|
||||
target.emitAndWaitConcurrently(`${key}.nobody`, {}, {}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('off', () => {
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
*/
|
||||
|
||||
import { extensionStore } from '../../extensions';
|
||||
import { BoundedTtlMap } from '../../util/boundedTtlMap.js';
|
||||
import { withSpan } from '../../util/span.js';
|
||||
import { PuterClient } from '../types';
|
||||
import {
|
||||
@@ -29,13 +28,8 @@ import {
|
||||
MatchingEvents,
|
||||
} from './types';
|
||||
|
||||
const NO_LISTENERS: readonly EventListener[] = [];
|
||||
|
||||
export class EventClient extends PuterClient {
|
||||
#eventListeners: Partial<Record<ListenKey, EventListener[]>> = {};
|
||||
#matchKeyMemo = new BoundedTtlMap<string, readonly ListenKey[]>({
|
||||
maxEntries: 1024,
|
||||
});
|
||||
|
||||
onServerStart() {
|
||||
this.emit('serverStart', {}, {});
|
||||
@@ -69,8 +63,19 @@ export class EventClient extends PuterClient {
|
||||
data: EventMap[T],
|
||||
meta: EventMetadata,
|
||||
) {
|
||||
for (const matchKey of this.#matchKeys(key)) {
|
||||
for (const listener of this.#listenersFor(matchKey)) {
|
||||
const parts = key.split('.');
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const matchKey = (
|
||||
i === parts.length - 1
|
||||
? key
|
||||
: `${parts.slice(0, i + 1).join('.')}.*`
|
||||
) as ListenKey;
|
||||
const extensionListeners = extensionStore.events[matchKey];
|
||||
const listeners = (this.#eventListeners[matchKey] || []).concat(
|
||||
extensionListeners || [],
|
||||
);
|
||||
if (!listeners) continue;
|
||||
for (const listener of listeners) {
|
||||
this.#emitEvent(listener, key, data, meta);
|
||||
}
|
||||
}
|
||||
@@ -87,27 +92,32 @@ export class EventClient extends PuterClient {
|
||||
*
|
||||
* Listeners run sequentially in the order they're registered so an earlier
|
||||
* handler's mutation is visible to later ones. A listener that throws is
|
||||
* logged and the chain continues, with `meta.listener_failed` set.
|
||||
* logged (same as `emit`) and the chain continues.
|
||||
*/
|
||||
async emitAndWait<T extends keyof EventMap>(
|
||||
key: T,
|
||||
data: EventMap[T],
|
||||
meta: EventMetadata,
|
||||
) {
|
||||
if (!this.hasListeners(key)) return;
|
||||
// Spanned because callers block on listeners — this is where time
|
||||
// spent in extension hooks (e.g. `ip.validate`) gets attributed.
|
||||
return withSpan('event.emitAndWait', { 'event.key': key }, async () => {
|
||||
for (const matchKey of this.#matchKeys(key)) {
|
||||
// A wildcard listener observes the event; it does not implement it.
|
||||
const isExact = matchKey === key;
|
||||
for (const listener of this.#listenersFor(matchKey)) {
|
||||
const parts = key.split('.');
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const matchKey = (
|
||||
i === parts.length - 1
|
||||
? key
|
||||
: `${parts.slice(0, i + 1).join('.')}.*`
|
||||
) as ListenKey;
|
||||
const extensionListeners = extensionStore.events[matchKey];
|
||||
const listeners = (this.#eventListeners[matchKey] || []).concat(
|
||||
extensionListeners || [],
|
||||
);
|
||||
if (!listeners) continue;
|
||||
for (const listener of listeners) {
|
||||
try {
|
||||
await listener(key, data, meta);
|
||||
} catch (e) {
|
||||
// The chain continues, but the caller is told: a hook
|
||||
// that could not run is not a hook that approved.
|
||||
if (isExact) meta.listener_failed = true;
|
||||
console.error(
|
||||
'Error in event listener for event',
|
||||
key,
|
||||
@@ -119,43 +129,6 @@ export class EventClient extends PuterClient {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Like `emitAndWait`, but every matched listener runs at once, so none sees
|
||||
* another's mutation of `data` before all have finished. Only for events
|
||||
* whose listeners each add an independent share, written after their own
|
||||
* awaits (`const v = await x; data.n += v`).
|
||||
*/
|
||||
async emitAndWaitConcurrently<T extends keyof EventMap>(
|
||||
key: T,
|
||||
data: EventMap[T],
|
||||
meta: EventMetadata,
|
||||
) {
|
||||
const matched = this.#matchKeys(key).flatMap((matchKey) =>
|
||||
this.#listenersFor(matchKey).map((listener) => ({
|
||||
listener,
|
||||
// A wildcard listener observes the event; it does not implement it.
|
||||
isExact: matchKey === key,
|
||||
})),
|
||||
);
|
||||
if (matched.length === 0) return;
|
||||
return withSpan('event.emitAndWait', { 'event.key': key }, async () => {
|
||||
await Promise.all(
|
||||
matched.map(async ({ listener, isExact }) => {
|
||||
try {
|
||||
await listener(key, data, meta);
|
||||
} catch (e) {
|
||||
if (isExact) meta.listener_failed = true;
|
||||
console.error(
|
||||
'Error in event listener for event',
|
||||
key,
|
||||
e,
|
||||
);
|
||||
}
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether anything would run if `key` were emitted, wildcards included.
|
||||
*
|
||||
@@ -164,7 +137,13 @@ export class EventClient extends PuterClient {
|
||||
* otherwise perfectly cheap and doesn't need this.
|
||||
*/
|
||||
hasListeners<T extends keyof EventMap>(key: T): boolean {
|
||||
for (const matchKey of this.#matchKeys(key)) {
|
||||
const parts = key.split('.');
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const matchKey = (
|
||||
i === parts.length - 1
|
||||
? key
|
||||
: `${parts.slice(0, i + 1).join('.')}.*`
|
||||
) as ListenKey;
|
||||
if (this.#eventListeners[matchKey]?.length) return true;
|
||||
if (extensionStore.events[matchKey]?.length) return true;
|
||||
}
|
||||
@@ -207,34 +186,6 @@ export class EventClient extends PuterClient {
|
||||
const idx = listeners.indexOf(callback as EventListener);
|
||||
if (idx !== -1) listeners.splice(idx, 1);
|
||||
}
|
||||
/**
|
||||
* The keys a listener can be registered under to hear `key`: each
|
||||
* `<prefix>.*` shorter than the key, then the key itself. Memoized, since
|
||||
* the same few keys are emitted on every request.
|
||||
*/
|
||||
#matchKeys(key: string): readonly ListenKey[] {
|
||||
const cached = this.#matchKeyMemo.get(key);
|
||||
if (cached) return cached;
|
||||
const parts = key.split('.');
|
||||
const matchKeys = parts.map(
|
||||
(_, i) =>
|
||||
(i === parts.length - 1
|
||||
? key
|
||||
: `${parts.slice(0, i + 1).join('.')}.*`) as ListenKey,
|
||||
);
|
||||
this.#matchKeyMemo.set(key, matchKeys);
|
||||
return matchKeys;
|
||||
}
|
||||
|
||||
#listenersFor(matchKey: ListenKey): readonly EventListener[] {
|
||||
const own = this.#eventListeners[matchKey];
|
||||
const fromExtensions = extensionStore.events[matchKey];
|
||||
if (!own?.length && !fromExtensions?.length) return NO_LISTENERS;
|
||||
// A copy, so a listener that unsubscribes mid-dispatch can't make the
|
||||
// loop skip its neighbour.
|
||||
return (own ?? []).concat(fromExtensions ?? []);
|
||||
}
|
||||
|
||||
async #emitEvent<T extends keyof EventMap>(
|
||||
listener: EventListener,
|
||||
key: T,
|
||||
|
||||
@@ -175,7 +175,6 @@ export type EventMap = {
|
||||
index_url?: string | null;
|
||||
owner_user_id?: number | null;
|
||||
name?: string | null;
|
||||
godmode?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -268,17 +267,6 @@ export type EventMap = {
|
||||
message: string | null;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
// Inbound mail to a feedback address (`abuse@`, `fbl@`, ...), emitted via
|
||||
// `emitAndWait`. A listener that takes it sets `handled` and it reaches no
|
||||
// mailbox; otherwise it is routed like any other message.
|
||||
'email.ingress.feedback': {
|
||||
/** The envelope recipient, as the relay presented it. */
|
||||
to: string;
|
||||
from: string | null;
|
||||
/** The whole message, exactly as received. */
|
||||
raw: Buffer;
|
||||
handled: boolean;
|
||||
};
|
||||
'user.save_account': {
|
||||
user_id: number;
|
||||
old_username?: string;
|
||||
@@ -305,9 +293,6 @@ export type EventMap = {
|
||||
device_fingerprint: string | null;
|
||||
allowed: boolean;
|
||||
reason: string | null;
|
||||
// With `allowed` false, offer the card fallback in place of SMS. Only
|
||||
// honored while the fallback is on; otherwise the send proceeds.
|
||||
card_fallback_instead?: boolean;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
// Fire-and-forget signal that a code was actually sent — the abuse
|
||||
@@ -318,9 +303,6 @@ export type EventMap = {
|
||||
user_uid: string;
|
||||
phone: string;
|
||||
device_fingerprint: string | null;
|
||||
// Browser-signals dispatch id the client sent with this request, or
|
||||
// null when it sent none.
|
||||
dispatch_id: string | null;
|
||||
};
|
||||
'user.phone-verified': {
|
||||
user_id: number;
|
||||
@@ -340,9 +322,6 @@ export type EventMap = {
|
||||
// the abuse extension cap card-verification setups per device (across
|
||||
// accounts) before any Stripe SetupIntent is created.
|
||||
device_fingerprint: string | null;
|
||||
// 'trial' when the card is being added for a free trial; null for the
|
||||
// verification gates.
|
||||
purpose?: string | null;
|
||||
enabled: boolean | null;
|
||||
// Set false by the extension to refuse this setup (e.g. the per-device
|
||||
// setup-velocity cap); `reason` carries the opaque code. Stays true
|
||||
@@ -397,8 +376,6 @@ export type EventMap = {
|
||||
user_id: number;
|
||||
user_uuid?: string;
|
||||
stripe_customer_id?: string | null;
|
||||
/** Apps the account owned, which cascaded away with it. */
|
||||
appUids?: string[];
|
||||
};
|
||||
|
||||
// ---- Team billing ---- the trigger, never the charge ----
|
||||
@@ -461,11 +438,7 @@ export type EventMap = {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Bytes to add to a user's storage allowance. Listeners run concurrently:
|
||||
* add to `extra` after your own awaits, never `extra += await x`.
|
||||
*/
|
||||
'storage.quota.bonus': { userId: number; userUuid?: string; extra: number };
|
||||
'storage.quota.bonus': { userId: number; extra: number };
|
||||
|
||||
// ---- Key-value ----
|
||||
/**
|
||||
@@ -490,35 +463,13 @@ export type EventMap = {
|
||||
values?: unknown[];
|
||||
/** Keys among `keys` private to the namespace's app. */
|
||||
noShareKeys?: string[];
|
||||
/** Events handler runs behind the write; see `Actor.handlerDepth`. */
|
||||
handlerDepth?: number;
|
||||
/**
|
||||
* The user who made the change. Differs from `userId` when a share
|
||||
* holder wrote into the owner's namespace.
|
||||
*/
|
||||
actingUserId?: number;
|
||||
/** Made by a worker session or an events handler; see `viaWorkerOf`. */
|
||||
viaWorker?: true;
|
||||
};
|
||||
/**
|
||||
* A whole namespace was emptied. Namespace-level on purpose: `flush`'s own
|
||||
* key enumeration is truncated for a large namespace, so the keys it saw
|
||||
* are not the keys it removed.
|
||||
*/
|
||||
'kv.flushed': {
|
||||
namespace: string;
|
||||
userId: number;
|
||||
actingUserId?: number;
|
||||
viaWorker?: true;
|
||||
};
|
||||
/**
|
||||
* Share handles stopped being addresses, whoever retired them. `userId` is
|
||||
* one user who could have subscribed through them, their owner or their
|
||||
* holder. Listeners end what was made through them even where another grant
|
||||
* still covers the region. Local, like `permission.revoked`: one region
|
||||
* settling covers every other.
|
||||
*/
|
||||
'kv.share.retired': { handles: string[]; userId: number };
|
||||
'kv.flushed': { namespace: string; userId: number };
|
||||
|
||||
// ---- Notifications ----
|
||||
/**
|
||||
@@ -535,8 +486,6 @@ export type EventMap = {
|
||||
appUid: string | null;
|
||||
value: Record<string, unknown>;
|
||||
createdAt: number;
|
||||
/** Events handler runs behind whatever created it. */
|
||||
handlerDepth?: number;
|
||||
};
|
||||
/**
|
||||
* One notification, addressed at whatever sockets each region holds for the
|
||||
@@ -640,11 +589,6 @@ export type EventMap = {
|
||||
generation: number;
|
||||
/** Whether the table changed; only then does a peer need to re-read it. */
|
||||
durable: boolean;
|
||||
/**
|
||||
* Session subscriptions this bump ended, so siblings stop queueing for
|
||||
* them.
|
||||
*/
|
||||
ended?: string[];
|
||||
};
|
||||
/**
|
||||
* A user's sockets moved between regions, so every node must drop what it
|
||||
@@ -659,12 +603,6 @@ export type EventMap = {
|
||||
userId: number;
|
||||
generation: number;
|
||||
};
|
||||
/**
|
||||
* Someone's entry was just shared, so every process must drop its cached
|
||||
* "this owner has nothing shared" answer. `outer.pubsub.*` for the same
|
||||
* reason as the events generation bump: a per-process cache.
|
||||
*/
|
||||
'outer.pubsub.share.ownerShared': { ownerUserId: number };
|
||||
'outer.fs.write-hash': { hash: string; uuid: string };
|
||||
/**
|
||||
* Cache keys the KV read cache must stop serving, because the entries
|
||||
@@ -701,10 +639,6 @@ export type EventMap = {
|
||||
requestHash?: string;
|
||||
mime: string;
|
||||
};
|
||||
// Asked for a request whose host is outside every configured domain. A
|
||||
// listener that recognises the host sets `result.subdomain` to the site
|
||||
// it serves; an unclaimed host is answered with a 404.
|
||||
'site.resolveHost': { host: string; result: { subdomain: string | null } };
|
||||
// Asked before a hosted file is streamed. A listener that owns the site
|
||||
// sets `result.allowed = false` to withhold the entry; the visitor then
|
||||
// sees the same 404 as for a missing file.
|
||||
@@ -726,8 +660,7 @@ export type EventMap = {
|
||||
uid?: string;
|
||||
thumbnail?: string | null;
|
||||
};
|
||||
/** `uuid` is the entry the thumbnail belongs to. */
|
||||
'thumbnail.created': { url: string; uuid: string };
|
||||
'thumbnail.created': { url: string };
|
||||
'thumbnail.upload.prepare': {
|
||||
items: { index: number; item_uid: string }[];
|
||||
uploadUrl?: string;
|
||||
@@ -742,19 +675,8 @@ export type EventMap = {
|
||||
* Sessions were revoked for a user. Anything holding one of them open — a
|
||||
* live socket, say — should drop it: `revoked_at` is otherwise only read on
|
||||
* the next handshake.
|
||||
*
|
||||
* `outer.*` rather than `outer.pubsub.*` on purpose: the eviction runs
|
||||
* through the socket.io Redis adapter a cluster shares, so one node
|
||||
* disconnecting covers every node in it — fanning out to siblings would
|
||||
* just have each of them repeat the same cluster-wide disconnect.
|
||||
*/
|
||||
'outer.auth.sessions.revoked': {
|
||||
user_id: number;
|
||||
session_uids: string[];
|
||||
};
|
||||
|
||||
/** One access token was revoked; only its own connections should drop. */
|
||||
'auth.access-token.revoked': { token_uid: string };
|
||||
'auth.sessions.revoked': { user_id: number; session_uids: string[] };
|
||||
|
||||
/**
|
||||
* A grant was withdrawn, so whatever was standing on it has to be settled
|
||||
@@ -786,7 +708,6 @@ export type EventMap = {
|
||||
email?: string;
|
||||
app_name?: string;
|
||||
};
|
||||
/** False for an app actor: add only `subscribed` / `paid_storage` then. */
|
||||
isUser: boolean;
|
||||
};
|
||||
'wisp.get-policy': {
|
||||
@@ -821,12 +742,6 @@ export type EventMap = {
|
||||
// `ai.cost.factor.<driverName>.<provider>:<model>`. Emitted once per model
|
||||
// per batch; the last listener to set `factor` wins.
|
||||
[K in `ai.cost.factor.${string}`]: AiCostFactorEvent;
|
||||
} & {
|
||||
// A plan gate is about to refuse a caller. Keyed per surface the way the
|
||||
// lifecycles above are: `subscription.gate.route.<method>.<path>`,
|
||||
// `subscription.gate.driver.<iface>.<method>`. Emitted via `emitAndWait`;
|
||||
// a listener that sets `allow` waives the plan requirement for that call.
|
||||
[K in `subscription.gate.${string}`]: SubscriptionGateEvent;
|
||||
} & {
|
||||
[K in `pubsub.login.${string}`]: { authtoken: string };
|
||||
} & {
|
||||
@@ -847,18 +762,6 @@ export type EventMap = {
|
||||
'outer.pubsub.metering.credits-changed': { userUuid: string };
|
||||
} & IExtensionEventMap;
|
||||
|
||||
/** Payload for `subscription.gate.<surface>` events. */
|
||||
export type SubscriptionGateEvent = {
|
||||
actor: Actor;
|
||||
/** The key after `subscription.gate.`, e.g. `driver.puter-kvstore.get`. */
|
||||
surface: string;
|
||||
/** What the gate asked for: any paid plan (`true`) or these policy ids. */
|
||||
requirement: boolean | readonly string[];
|
||||
/** The request being gated, when there is one. */
|
||||
req?: ExpressRequest;
|
||||
allow: boolean;
|
||||
};
|
||||
|
||||
/** Payload for `ai.cost.factor.<driver>.<model>` events. */
|
||||
export type AiCostFactorEvent = {
|
||||
/** Driver doing the pricing, e.g. `ai-chat`. */
|
||||
@@ -949,11 +852,10 @@ export type EventKey = keyof EventMap & string;
|
||||
// Generates a wildcard for every non-final dot-separated prefix of K.
|
||||
export type WildcardPrefixes<K extends string> =
|
||||
K extends `${infer Head}.${infer Tail}`
|
||||
?
|
||||
| `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
? | `${Head}.*`
|
||||
| (Tail extends `${string}.${string}`
|
||||
? `${Head}.${WildcardPrefixes<Tail>}`
|
||||
: never)
|
||||
: never;
|
||||
|
||||
export type ListenKey = EventKey | WildcardPrefixes<EventKey>;
|
||||
@@ -962,11 +864,7 @@ export type MatchingEvents<P extends ListenKey> = P extends `${infer Prefix}.*`
|
||||
? Extract<EventKey, `${Prefix}.${string}`>
|
||||
: P & EventKey;
|
||||
|
||||
export type EventMetadata = {
|
||||
from_outside?: boolean;
|
||||
/** Set by `emitAndWait` when a listener threw, so a verdict hook can refuse. */
|
||||
listener_failed?: boolean;
|
||||
};
|
||||
export type EventMetadata = { from_outside?: boolean };
|
||||
export type EventListener<K extends EventKey = EventKey> = (
|
||||
key: K,
|
||||
data: EventMap[K],
|
||||
|
||||
@@ -3,19 +3,18 @@
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify it under the
|
||||
* terms of the GNU Affero General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||
* details.
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see
|
||||
* [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { GetObjectCommand } from '@aws-sdk/client-s3';
|
||||
@@ -25,7 +24,7 @@ import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { IConfig } from '../../types';
|
||||
import { S3Client, withPresignHeadroom } from './S3Client';
|
||||
import { S3Client } from './S3Client';
|
||||
|
||||
const LOCAL_BUCKET = 'puter-local';
|
||||
|
||||
@@ -488,54 +487,3 @@ describe('S3Client — short reads during multipart', () => {
|
||||
expect(parts).toEqual([10, 10]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('withPresignHeadroom', () => {
|
||||
const HOUR = 60 * 60 * 1000;
|
||||
const creds = (msLeft?: number, id = 'a') => ({
|
||||
accessKeyId: id,
|
||||
secretAccessKey: 's',
|
||||
...(msLeft === undefined
|
||||
? {}
|
||||
: { expiration: new Date(Date.now() + msLeft) }),
|
||||
});
|
||||
|
||||
it('reports expiry an hour early so the SDK refreshes while an hour remains', async () => {
|
||||
const chain = vi.fn(async () => creds(5 * HOUR));
|
||||
const identity = await withPresignHeadroom(chain)();
|
||||
|
||||
expect(chain).toHaveBeenCalledTimes(1);
|
||||
const left = identity.expiration!.getTime() - Date.now();
|
||||
expect(left).toBeGreaterThan(4 * HOUR - 1000);
|
||||
expect(left).toBeLessThanOrEqual(4 * HOUR);
|
||||
});
|
||||
|
||||
it('force-refreshes credentials that are inside the presign window', async () => {
|
||||
const chain = vi.fn(async (options?: { forceRefresh?: boolean }) =>
|
||||
options?.forceRefresh
|
||||
? creds(6 * HOUR, 'fresh')
|
||||
: creds(30 * 60 * 1000, 'stale'),
|
||||
);
|
||||
const identity = await withPresignHeadroom(chain)();
|
||||
|
||||
expect(chain).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ forceRefresh: true }),
|
||||
);
|
||||
expect(identity.accessKeyId).toBe('fresh');
|
||||
});
|
||||
|
||||
it('leaves long-lived credentials alone', async () => {
|
||||
const chain = vi.fn(async () => creds(undefined));
|
||||
const identity = await withPresignHeadroom(chain)();
|
||||
|
||||
expect(chain).toHaveBeenCalledTimes(1);
|
||||
expect(identity.expiration).toBeUndefined();
|
||||
});
|
||||
|
||||
it('never reports an expiry inside the SDK refresh margin', async () => {
|
||||
const chain = vi.fn(async () => creds(20 * 60 * 1000));
|
||||
const identity = await withPresignHeadroom(chain)();
|
||||
|
||||
const left = identity.expiration!.getTime() - Date.now();
|
||||
expect(left).toBeGreaterThan(5 * 60 * 1000);
|
||||
});
|
||||
});
|
||||
@@ -43,43 +43,6 @@ const DEFAULT_MULTIPART_PART_SIZE_BYTES = 5 * 1024 * 1024;
|
||||
const FAUXQS_SAFE_PUT_OBJECT_LIMIT_BYTES = 10 * 1024 * 1024;
|
||||
const LEGACY_STORAGE_BUCKET = 'puter-local';
|
||||
|
||||
// A presigned URL stops working when the credentials that signed it expire.
|
||||
// Presigns last up to an hour, but the SDK only refreshes temporary
|
||||
// credentials five minutes before expiry.
|
||||
const PRESIGN_MAX_LIFETIME_MS = 60 * 60 * 1000;
|
||||
const SDK_REFRESH_MARGIN_MS = 5 * 60 * 1000;
|
||||
|
||||
type CredentialChain = ReturnType<typeof fromNodeProviderChain>;
|
||||
|
||||
/**
|
||||
* Keeps temporary credentials at least an hour from expiry when they sign. The
|
||||
* SDK is handed each identity with its expiration moved an hour early, so its
|
||||
* own refresh comes back here while an hour remains, and a credential already
|
||||
* inside that window is force-refreshed from the chain.
|
||||
*/
|
||||
export function withPresignHeadroom(chain: CredentialChain): CredentialChain {
|
||||
return async (options) => {
|
||||
let identity = await chain(options);
|
||||
const remaining = (i: typeof identity) =>
|
||||
i.expiration ? i.expiration.getTime() - Date.now() : Infinity;
|
||||
if (
|
||||
remaining(identity) <
|
||||
PRESIGN_MAX_LIFETIME_MS + SDK_REFRESH_MARGIN_MS
|
||||
) {
|
||||
identity = await chain({ ...options, forceRefresh: true });
|
||||
}
|
||||
if (!identity.expiration) return identity;
|
||||
// If even fresh credentials are short-lived, report an expiry just past
|
||||
// the SDK's margin rather than one in the past, which would make it
|
||||
// re-resolve on every request.
|
||||
const reported = Math.max(
|
||||
identity.expiration.getTime() - PRESIGN_MAX_LIFETIME_MS,
|
||||
Date.now() + SDK_REFRESH_MARGIN_MS + 60 * 1000,
|
||||
);
|
||||
return { ...identity, expiration: new Date(reported) };
|
||||
};
|
||||
}
|
||||
|
||||
type S3CommandSender = Pick<AwsS3Client, 'send'>;
|
||||
|
||||
export class S3Client extends PuterClient {
|
||||
@@ -120,9 +83,7 @@ export class S3Client extends PuterClient {
|
||||
|
||||
if (useCredentialChain) {
|
||||
this.useProviderChain = true;
|
||||
this.awsConfig = {
|
||||
credentials: withPresignHeadroom(fromNodeProviderChain()),
|
||||
};
|
||||
this.awsConfig = { credentials: fromNodeProviderChain() };
|
||||
this.partSize = 64 * 1024 * 1024;
|
||||
this.maxSingleUploadSize = 128 * 1024 * 1024;
|
||||
} else {
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { IConfig } from './types';
|
||||
|
||||
// Config resolution order:
|
||||
// 1. `process.env.PUTER_CONFIG_PATH` — absolute path to a config file. Used
|
||||
// by deployments whose outer bootstrap writes a merged config to a known
|
||||
// location.
|
||||
// 2. `<PACKAGE_ROOT>/config.json` — user's runtime override (gitignored),
|
||||
// deep-merged over config.default.json so users can omit keys they
|
||||
// don't care to override (e.g. gui_assets_root, database).
|
||||
// 3. `<PACKAGE_ROOT>/config.default.json` — bundled OSS defaults.
|
||||
//
|
||||
// Post-flatten depth: compiled file is at `packages/puter/dist/src/backend/index.js`,
|
||||
// so three `..`s land at `packages/puter/`.
|
||||
const PACKAGE_ROOT = path.resolve(__dirname, '../../..');
|
||||
// Root of the running code tree. Matches PACKAGE_ROOT for a source run, but
|
||||
// points at `dist/` for a compiled run — so config-declared paths like
|
||||
// `./extensions` resolve to `dist/extensions` at runtime without the config
|
||||
// having to know about the build layout.
|
||||
const RUNTIME_ROOT = path.resolve(__dirname, '../..');
|
||||
|
||||
const isPlainObject = (v: unknown): v is Record<string, unknown> =>
|
||||
typeof v === 'object' && v !== null && !Array.isArray(v);
|
||||
|
||||
const deepMerge = <T extends Record<string, unknown>>(
|
||||
base: T,
|
||||
override: Record<string, unknown>,
|
||||
): T => {
|
||||
const out: Record<string, unknown> = { ...base };
|
||||
for (const [k, v] of Object.entries(override)) {
|
||||
out[k] =
|
||||
isPlainObject(v) && isPlainObject(out[k])
|
||||
? deepMerge(out[k] as Record<string, unknown>, v)
|
||||
: v;
|
||||
}
|
||||
return out as T;
|
||||
};
|
||||
|
||||
export const loadConfig = (): IConfig => {
|
||||
const envPath = process.env.PUTER_CONFIG_PATH;
|
||||
const runtimePath = path.join(PACKAGE_ROOT, 'config.json');
|
||||
const defaultPath = path.join(PACKAGE_ROOT, 'config.default.json');
|
||||
|
||||
const defaults = existsSync(defaultPath)
|
||||
? (JSON.parse(readFileSync(defaultPath, 'utf8')) as Record<
|
||||
string,
|
||||
unknown
|
||||
>)
|
||||
: {};
|
||||
|
||||
// Runtime override path: env wins, then config.json, else no override
|
||||
// (we still return defaults so single-file installs work).
|
||||
const overridePath =
|
||||
envPath && existsSync(envPath)
|
||||
? envPath
|
||||
: existsSync(runtimePath)
|
||||
? runtimePath
|
||||
: null;
|
||||
|
||||
console.log(`[config] defaults from ${defaultPath}`);
|
||||
if (overridePath) console.log(`[config] override from ${overridePath}`);
|
||||
|
||||
const override = overridePath
|
||||
? (JSON.parse(readFileSync(overridePath, 'utf8')) as Record<
|
||||
string,
|
||||
unknown
|
||||
>)
|
||||
: {};
|
||||
|
||||
const config = deepMerge(defaults, override) as IConfig;
|
||||
|
||||
if (!config.version) {
|
||||
const pkgPath = path.join(PACKAGE_ROOT, 'package.json');
|
||||
if (existsSync(pkgPath)) {
|
||||
try {
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as {
|
||||
version?: string;
|
||||
};
|
||||
if (pkg.version) config.version = pkg.version;
|
||||
} catch {
|
||||
// fall through — /version returns 'unknown'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Computed defaults. `origin` and `pub_port` are the externally-visible
|
||||
// URL+port — what the browser sees. Separate from `port`, which is the
|
||||
// bind port (can differ when behind a reverse proxy). Code paths that
|
||||
// build self-referential URLs (GUI bootstrap, email links, OIDC callbacks)
|
||||
// depend on `origin` having the right port baked in.
|
||||
if (config.pub_port === undefined) config.pub_port = config.port;
|
||||
const protocol = config.protocol ?? 'http';
|
||||
const domain = config.domain ?? 'localhost';
|
||||
const portSuffix =
|
||||
config.pub_port === 80 || config.pub_port === 443
|
||||
? ''
|
||||
: `:${config.pub_port}`;
|
||||
if (config.origin === undefined) {
|
||||
config.origin = `${protocol}://${domain}${portSuffix}`;
|
||||
}
|
||||
// API lives on the `api.` subdomain on the same host+port as the main
|
||||
// origin (see PuterRouter subdomain handling in server.ts). Compute it
|
||||
// from pub_port/domain so a single-port override (e.g. port: 5101) flows
|
||||
// through to the GUI bootstrap without users having to restate the URL.
|
||||
if (config.api_base_url === undefined) {
|
||||
config.api_base_url = `${protocol}://api.${domain}${portSuffix}`;
|
||||
}
|
||||
|
||||
// Resolve path-valued config fields. Two different roots:
|
||||
// - `extensions` uses RUNTIME_ROOT because extensions ship inside the
|
||||
// build output (dist/extensions) and the loader's dynamic import()
|
||||
// resolves relative paths against the *importing* module file.
|
||||
// - GUI/puter-js/builtin-apps use PACKAGE_ROOT because those assets
|
||||
// live only in the source tree (not copied into dist/) and are served
|
||||
// via express.static at runtime.
|
||||
const resolveRuntime = (p: string): string =>
|
||||
path.isAbsolute(p) ? p : path.resolve(RUNTIME_ROOT, p);
|
||||
const resolvePackage = (p: string): string =>
|
||||
path.isAbsolute(p) ? p : path.resolve(PACKAGE_ROOT, p);
|
||||
|
||||
if (Array.isArray(config.extensions)) {
|
||||
config.extensions = config.extensions.map(resolveRuntime);
|
||||
}
|
||||
if (typeof config.gui_assets_root === 'string') {
|
||||
config.gui_assets_root = resolvePackage(config.gui_assets_root);
|
||||
}
|
||||
if (typeof config.puterjs_root === 'string') {
|
||||
config.puterjs_root = resolvePackage(config.puterjs_root);
|
||||
}
|
||||
if (isPlainObject(config.builtin_apps)) {
|
||||
for (const [k, v] of Object.entries(config.builtin_apps)) {
|
||||
if (typeof v === 'string') {
|
||||
(config.builtin_apps as Record<string, string>)[k] =
|
||||
resolvePackage(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
return config;
|
||||
};
|
||||
@@ -17,21 +17,25 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import {
|
||||
isAccessTokenActor,
|
||||
isAccountContext,
|
||||
isAppActor,
|
||||
} from '../../core/actor.js';
|
||||
import { isAccessTokenActor, isAppActor } from '../../core/actor.js';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import { driversContainers } from '../../exports.js';
|
||||
import {
|
||||
APP_ICON_SIZES,
|
||||
ICON_DATA_URL_MIME_ALLOWLIST,
|
||||
isTrustedIconHost,
|
||||
} from '../../util/appIcon.js';
|
||||
import { toAppReadView } from '../../util/appView.js';
|
||||
import { resolvePrivateLaunchAccess } from '../../util/privateLaunchAccess.js';
|
||||
import { PuterController } from '../types.js';
|
||||
import DEFAULT_APP_ICON from './default-app-icon.js';
|
||||
|
||||
/**
|
||||
* REST endpoints for app management.
|
||||
*
|
||||
* Delegates to AppDriver for the actual CRUD + permission logic — these routes
|
||||
* are just thin shape adapters that translate REST conventions into driver
|
||||
* calls.
|
||||
*/
|
||||
/**
|
||||
* Desktop boot reads the app list and individual app records repeatedly, so the
|
||||
* ceiling is set well above normal boot traffic and exists to catch a runaway
|
||||
@@ -65,19 +69,11 @@ const APP_ICON_LIMIT = {
|
||||
key: 'ip',
|
||||
};
|
||||
|
||||
// Shared by `GET /apps/:name` (pipe-separated) and `POST /query/app`, which
|
||||
// both fan out one lookup per name.
|
||||
const APP_LOOKUP_MAX_ENTRIES = 200;
|
||||
const APP_LOOKUP_MAX_SELECTOR_LEN = 200;
|
||||
|
||||
/** What `/apps/:name` reports for an app no launch gate applies to. */
|
||||
const PUBLIC_APP_ACCESS = { hasAccess: true, checkedBy: 'core/public-app' };
|
||||
|
||||
/**
|
||||
* REST endpoints for app management. CRUD goes through the apps driver; the
|
||||
* read routes use AppService for visibility and views, as the driver does.
|
||||
*/
|
||||
export class AppController extends PuterController {
|
||||
get appStore() {
|
||||
return this.stores.app;
|
||||
}
|
||||
|
||||
// In-flight background app-open writes. Tracked only so tests and
|
||||
// shutdown can wait for them — the request path never does.
|
||||
#pendingOpenWrites = new Set();
|
||||
@@ -129,6 +125,15 @@ export class AppController extends PuterController {
|
||||
await Promise.allSettled([...this.#pendingOpenWrites]);
|
||||
}
|
||||
|
||||
get appDriver() {
|
||||
// Drivers are wired into the shared driversContainers export by
|
||||
// PuterServer at boot. Controllers get them lazily via this getter
|
||||
// since they're instantiated before drivers in the boot order.
|
||||
const d = driversContainers.apps;
|
||||
if (!d) throw new Error('AppDriver not registered yet');
|
||||
return d;
|
||||
}
|
||||
|
||||
registerRoutes(
|
||||
/** @type {import('../../core/http/PuterRouter').PuterRouter} */ router,
|
||||
) {
|
||||
@@ -142,7 +147,7 @@ export class AppController extends PuterController {
|
||||
rateLimit: APP_READ_LIMIT,
|
||||
},
|
||||
async (req, res) => {
|
||||
const apps = await this.drivers.apps.select({
|
||||
const apps = await this.appDriver.select({
|
||||
predicate: ['user-can-edit'],
|
||||
});
|
||||
res.json(apps);
|
||||
@@ -174,7 +179,7 @@ export class AppController extends PuterController {
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
const available = await this.drivers.apps.isNameAvailable(name);
|
||||
const available = await this.appDriver.isNameAvailable(name);
|
||||
res.json({ name, available });
|
||||
},
|
||||
);
|
||||
@@ -185,10 +190,10 @@ export class AppController extends PuterController {
|
||||
// before 400-ing for a missing body field.
|
||||
//
|
||||
// Authorization: only two callers are trusted to report opens —
|
||||
// 1. the account acting as itself (plain session or full-access
|
||||
// 1. a root user actor (plain session, no `.app` and no access
|
||||
// token), e.g. the GUI launching apps on behalf of the user;
|
||||
// 2. the app-under-user actor for the app being reported.
|
||||
// Everything else — scoped and app-issued access tokens, asset
|
||||
// Everything else — access tokens (regardless of issuer), asset
|
||||
// tokens, app actors reporting for a *different* app — is denied,
|
||||
// otherwise any authenticated party could inflate another app's
|
||||
// open count.
|
||||
@@ -213,11 +218,14 @@ export class AppController extends PuterController {
|
||||
});
|
||||
}
|
||||
|
||||
// Shared credentials shouldn't drive analytics counters.
|
||||
if (isAccessTokenActor(actor) && !isAccountContext(actor)) {
|
||||
// Access tokens (and any other non-user/non-app identity,
|
||||
// e.g. asset tokens) are not allowed to report opens —
|
||||
// they're shared / scoped credentials and shouldn't drive
|
||||
// analytics counters.
|
||||
if (isAccessTokenActor(actor)) {
|
||||
throw new HttpError(
|
||||
403,
|
||||
'This token cannot report app opens',
|
||||
'Access tokens cannot report app opens',
|
||||
{ legacyCode: 'forbidden' },
|
||||
);
|
||||
}
|
||||
@@ -230,7 +238,7 @@ export class AppController extends PuterController {
|
||||
);
|
||||
}
|
||||
|
||||
const app = await this.stores.app.getByUid(app_uid);
|
||||
const app = await this.appStore.getByUid(app_uid);
|
||||
if (!app)
|
||||
throw new HttpError(404, 'App not found', {
|
||||
legacyCode: 'not_found',
|
||||
@@ -258,40 +266,37 @@ export class AppController extends PuterController {
|
||||
async (req, res) => {
|
||||
const raw = req.params.name;
|
||||
const names = raw.split('|').filter(Boolean);
|
||||
if (names.length > APP_LOOKUP_MAX_ENTRIES) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`at most ${APP_LOOKUP_MAX_ENTRIES} app names per request`,
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
|
||||
const actor = req.actor;
|
||||
const appsByName = await this.stores.app.getByNames(
|
||||
names.filter(
|
||||
(name) => name.length <= APP_LOOKUP_MAX_SELECTOR_LEN,
|
||||
),
|
||||
);
|
||||
const readable = await this.services.app.filterReadable(
|
||||
[...new Set(appsByName.values())],
|
||||
actor,
|
||||
);
|
||||
const views = await this.services.app.views(readable, actor, {
|
||||
source: 'appsRoute',
|
||||
});
|
||||
const shapedByUid = new Map(
|
||||
readable.map((app, i) => {
|
||||
const shaped = toAppReadView(app, views[i], {
|
||||
viewer: actor.user,
|
||||
const userUid = req.actor?.user?.uuid ?? null;
|
||||
|
||||
const results = await Promise.all(
|
||||
names.map(async (name) => {
|
||||
const app = await this.appStore.getByName(name);
|
||||
if (!app) return null;
|
||||
let shaped;
|
||||
try {
|
||||
shaped = await this.appDriver.read({
|
||||
uid: app.uid,
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const privateAccess = await resolvePrivateLaunchAccess({
|
||||
app: shaped,
|
||||
eventClient: this.clients.event,
|
||||
userUid,
|
||||
source: 'appsRoute',
|
||||
args: req.query ?? {},
|
||||
});
|
||||
shaped.privateAccess ??= PUBLIC_APP_ACCESS;
|
||||
return [app.uid, shaped];
|
||||
return {
|
||||
...shaped,
|
||||
privateAccess:
|
||||
shaped.privateAccess?.hasAccess === false
|
||||
? shaped.privateAccess
|
||||
: privateAccess,
|
||||
};
|
||||
}),
|
||||
);
|
||||
const results = names.map(
|
||||
(name) =>
|
||||
shapedByUid.get(appsByName.get(name)?.uid) ?? null,
|
||||
);
|
||||
|
||||
// Single-name requests return the app directly; batch returns an array
|
||||
if (names.length === 1) {
|
||||
@@ -311,8 +316,8 @@ export class AppController extends PuterController {
|
||||
//
|
||||
// Access rules: only apps the caller has a legitimate reason to
|
||||
// see are returned — public (`approved_for_listing`), owned by
|
||||
// the caller, or readable per `AppService.filterReadable` (for
|
||||
// protected apps, a granted permission). Everything else is
|
||||
// the caller, or explicitly accessible via AppDriver.read (for
|
||||
// protected apps with a granted permission). Everything else is
|
||||
// silently skipped so the endpoint can't be used to enumerate
|
||||
// existence of private / unapproved apps by guessing names.
|
||||
//
|
||||
@@ -322,6 +327,9 @@ export class AppController extends PuterController {
|
||||
// included for public/owned apps only, consistent with
|
||||
// marketplace semantics.
|
||||
|
||||
const QUERY_APP_MAX_ENTRIES = 200;
|
||||
const QUERY_APP_MAX_SELECTOR_LEN = 200;
|
||||
|
||||
router.post(
|
||||
'/query/app',
|
||||
{
|
||||
@@ -331,62 +339,71 @@ export class AppController extends PuterController {
|
||||
},
|
||||
async (req, res) => {
|
||||
const appList = Array.isArray(req.body) ? req.body : [];
|
||||
if (appList.length > APP_LOOKUP_MAX_ENTRIES) {
|
||||
if (appList.length > QUERY_APP_MAX_ENTRIES) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
`request body must contain at most ${APP_LOOKUP_MAX_ENTRIES} selectors`,
|
||||
`request body must contain at most ${QUERY_APP_MAX_ENTRIES} selectors`,
|
||||
{ legacyCode: 'bad_request' },
|
||||
);
|
||||
}
|
||||
|
||||
const actor = req.actor;
|
||||
const selectors = appList.filter(
|
||||
(selector) =>
|
||||
typeof selector === 'string' &&
|
||||
selector.length > 0 &&
|
||||
selector.length <= APP_LOOKUP_MAX_SELECTOR_LEN,
|
||||
);
|
||||
const isUid = (selector) => selector.startsWith('app-');
|
||||
const [byUid, byName] = await Promise.all([
|
||||
this.stores.app.getByUids(selectors.filter(isUid)),
|
||||
this.stores.app.getByNames(
|
||||
selectors.filter((s) => !isUid(s)),
|
||||
),
|
||||
]);
|
||||
const found = selectors
|
||||
.map((s) => (isUid(s) ? byUid.get(s) : byName.get(s)))
|
||||
.filter(Boolean);
|
||||
const actorUserId = req.actor?.user?.id ?? null;
|
||||
const results = [];
|
||||
|
||||
// Unapproved apps the caller doesn't own surface only when
|
||||
// the caller may read them (one batched grant check).
|
||||
const isListed = (app) =>
|
||||
app.owner_user_id === actor.user?.id ||
|
||||
Boolean(app.approved_for_listing);
|
||||
const readable = new Set(
|
||||
await this.services.app.filterReadable(
|
||||
[...new Set(found.filter((app) => !isListed(app)))],
|
||||
actor,
|
||||
),
|
||||
);
|
||||
const visible = found.filter(
|
||||
(app) => isListed(app) || readable.has(app),
|
||||
);
|
||||
const filetypesByAppId =
|
||||
await this.stores.app.getFiletypeAssociationsByIds(
|
||||
visible.map((app) => app.id),
|
||||
for (const selector of appList) {
|
||||
if (
|
||||
typeof selector !== 'string' ||
|
||||
selector.length === 0 ||
|
||||
selector.length > QUERY_APP_MAX_SELECTOR_LEN
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const isUid = selector.startsWith('app-');
|
||||
const app = isUid
|
||||
? await this.appStore.getByUid(selector)
|
||||
: await this.appStore.getByName(selector);
|
||||
if (!app) continue;
|
||||
|
||||
const isOwner =
|
||||
actorUserId !== null &&
|
||||
app.owner_user_id === actorUserId;
|
||||
const isApproved = Boolean(app.approved_for_listing);
|
||||
|
||||
if (!isOwner && !isApproved) {
|
||||
// Unapproved, non-owned — only surface if the
|
||||
// caller has an explicit grant (purchased /
|
||||
// permissioned). AppDriver.read enforces that
|
||||
// via #canReadApp; a thrown 403 means "not
|
||||
// accessible" and we treat it as "not found".
|
||||
try {
|
||||
const shaped = await this.appDriver.read({
|
||||
uid: app.uid,
|
||||
});
|
||||
if (!shaped) continue;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const assocRows = await this.clients.db.read(
|
||||
'SELECT `type` FROM `app_filetype_association` WHERE `app_id` = ?',
|
||||
[app.id],
|
||||
);
|
||||
|
||||
const results = visible.map((app) => ({
|
||||
uuid: app.uid,
|
||||
name: app.name,
|
||||
title: app.title,
|
||||
description: app.description,
|
||||
metadata: app.metadata,
|
||||
tags:
|
||||
typeof app.tags === 'string' ? app.tags.split(',') : [],
|
||||
created: app.timestamp,
|
||||
associations: filetypesByAppId.get(app.id) ?? [],
|
||||
}));
|
||||
results.push({
|
||||
uuid: app.uid,
|
||||
name: app.name,
|
||||
title: app.title,
|
||||
description: app.description,
|
||||
metadata: app.metadata,
|
||||
tags:
|
||||
typeof app.tags === 'string'
|
||||
? app.tags.split(',')
|
||||
: [],
|
||||
created: app.timestamp,
|
||||
associations: assocRows.map((r) => r.type),
|
||||
});
|
||||
}
|
||||
|
||||
res.json(results);
|
||||
},
|
||||
@@ -444,7 +461,7 @@ export class AppController extends PuterController {
|
||||
}
|
||||
if (!appUid.startsWith('app-')) appUid = `app-${appUid}`;
|
||||
|
||||
const app = await this.stores.app.getByUid(appUid);
|
||||
const app = await this.appStore.getByUid(appUid);
|
||||
const icon = app?.icon;
|
||||
|
||||
// If the icon isn't an inline data URL, try to serve it from the
|
||||
|
||||
@@ -338,42 +338,6 @@ describe('AppController POST /rao', () => {
|
||||
expect(rows).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps every open as a row, and counts one per account per hour', async () => {
|
||||
const owner = await makeUser();
|
||||
const other = await makeUser();
|
||||
const app = await createApp(owner.actor);
|
||||
|
||||
const open = async (user: { actor: Actor; userId: number }) => {
|
||||
const { res } = makeRes();
|
||||
await withActor(user.actor, () =>
|
||||
callRoute(
|
||||
'post',
|
||||
'/rao',
|
||||
makeReq({ body: { app_uid: app.uid }, actor: user.actor }),
|
||||
res,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
for (let i = 0; i < 5; i++) await open(owner);
|
||||
await open(other);
|
||||
await server.controllers.apps.drainPendingAppOpens();
|
||||
|
||||
// Every open is still a row: the recent-apps list orders on them.
|
||||
const rows = (await server.clients.db.read(
|
||||
'SELECT `user_id` FROM `app_opens` WHERE `app_uid` = ?',
|
||||
[app.uid],
|
||||
)) as Array<{ user_id: number }>;
|
||||
expect(rows).toHaveLength(6);
|
||||
|
||||
// The count is accounts-per-hour, so repeats do not move it.
|
||||
const stats = await server.stores.app.getAppsStats([app.uid]);
|
||||
expect(stats.get(app.uid)).toMatchObject({
|
||||
open_count: 2,
|
||||
user_count: 2,
|
||||
});
|
||||
});
|
||||
|
||||
// `app_opens` is analytics: the response carries nothing derived from it
|
||||
// and the client never awaits the post. Holding the response open for a
|
||||
// primary write put that write's latency inside the app launch it was
|
||||
@@ -574,57 +538,6 @@ describe('AppController GET /apps/:name', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('throws 400 when the pipe-batch exceeds the 200-name cap', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const { res } = makeRes();
|
||||
const names = new Array(201).fill('x').join('|');
|
||||
await expect(
|
||||
withActor(actor, () =>
|
||||
callRoute(
|
||||
'get',
|
||||
'/apps/:name',
|
||||
makeReq({ params: { name: names }, actor }),
|
||||
res,
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
});
|
||||
|
||||
it('accepts a pipe-batch at exactly the 200-name cap', async () => {
|
||||
const owner = await makeUser();
|
||||
const app = await createApp(owner.actor);
|
||||
const names = [app.name, ...new Array(199).fill('missing-app')].join(
|
||||
'|',
|
||||
);
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await withActor(owner.actor, () =>
|
||||
callRoute(
|
||||
'get',
|
||||
'/apps/:name',
|
||||
makeReq({ params: { name: names }, actor: owner.actor }),
|
||||
res,
|
||||
),
|
||||
);
|
||||
expect(Array.isArray(captured.body)).toBe(true);
|
||||
expect(captured.body as unknown[]).toHaveLength(200);
|
||||
});
|
||||
|
||||
it('throws 404 for a single name over the per-name length cap', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
withActor(actor, () =>
|
||||
callRoute(
|
||||
'get',
|
||||
'/apps/:name',
|
||||
makeReq({ params: { name: 'x'.repeat(201) }, actor }),
|
||||
res,
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 404 });
|
||||
});
|
||||
|
||||
it('omits the stale index_url for a non-owner', async () => {
|
||||
const owner = await makeUser();
|
||||
const stranger = await makeUser();
|
||||
@@ -654,48 +567,6 @@ describe('AppController GET /apps/:name', () => {
|
||||
expect(body.index_url).toBeUndefined();
|
||||
expect(body.privateAccess).toMatchObject({ hasAccess: false });
|
||||
});
|
||||
|
||||
it('asks the private-access hook once per private app and withholds its index_url', async () => {
|
||||
const owner = await makeUser();
|
||||
const stranger = await makeUser();
|
||||
const app = await createApp(owner.actor, { name: uniqueName('priv') });
|
||||
const other = await createApp(owner.actor, { name: uniqueName('pub') });
|
||||
await server.clients.db.write(
|
||||
'UPDATE `apps` SET `is_private` = 1 WHERE `uid` = ?',
|
||||
[app.uid],
|
||||
);
|
||||
await server.stores.app.invalidateByUid(app.uid as string);
|
||||
const emit = vi.spyOn(server.clients.event, 'emitAndWait');
|
||||
|
||||
try {
|
||||
const { res, captured } = makeRes();
|
||||
await withActor(stranger.actor, () =>
|
||||
callRoute(
|
||||
'get',
|
||||
'/apps/:name',
|
||||
makeReq({
|
||||
params: { name: `${app.name}|${other.name}` },
|
||||
actor: stranger.actor,
|
||||
}),
|
||||
res,
|
||||
),
|
||||
);
|
||||
const [priv, pub] = captured.body as Array<Record<string, unknown>>;
|
||||
expect(priv!.index_url).toBeUndefined();
|
||||
expect(priv!.privateAccess).toMatchObject({ hasAccess: false });
|
||||
expect(pub!.privateAccess).toEqual({
|
||||
hasAccess: true,
|
||||
checkedBy: 'core/public-app',
|
||||
});
|
||||
expect(
|
||||
emit.mock.calls.filter(
|
||||
(call) => call[0] === 'app.privateAccess.resolveLaunch',
|
||||
),
|
||||
).toHaveLength(1);
|
||||
} finally {
|
||||
emit.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /query/app ─────────────────────────────────────────────────
|
||||
@@ -870,61 +741,6 @@ describe('AppController POST /query/app', () => {
|
||||
expect(body).toHaveLength(1);
|
||||
expect(body[0]?.name).toBe(app.name);
|
||||
});
|
||||
|
||||
it('resolves every selector with batched lookups, not one per selector', async () => {
|
||||
const owner = await makeUser();
|
||||
const apps = await Promise.all(
|
||||
[0, 1, 2].map(() =>
|
||||
createApp(owner.actor, {
|
||||
name: uniqueName('batch'),
|
||||
filetype_associations: ['.batchq'],
|
||||
}),
|
||||
),
|
||||
);
|
||||
const appStore = server.stores.app;
|
||||
const spies = {
|
||||
getByName: vi.spyOn(appStore, 'getByName'),
|
||||
getByUid: vi.spyOn(appStore, 'getByUid'),
|
||||
getFiletypeAssociations: vi.spyOn(
|
||||
appStore,
|
||||
'getFiletypeAssociations',
|
||||
),
|
||||
getByNames: vi.spyOn(appStore, 'getByNames'),
|
||||
getByUids: vi.spyOn(appStore, 'getByUids'),
|
||||
getFiletypeAssociationsByIds: vi.spyOn(
|
||||
appStore,
|
||||
'getFiletypeAssociationsByIds',
|
||||
),
|
||||
};
|
||||
|
||||
try {
|
||||
const { res, captured } = makeRes();
|
||||
await withActor(owner.actor, () =>
|
||||
callRoute(
|
||||
'post',
|
||||
'/query/app',
|
||||
makeReq({
|
||||
body: [apps[0]!.name, apps[1]!.name, apps[2]!.uid],
|
||||
actor: owner.actor,
|
||||
}),
|
||||
res,
|
||||
),
|
||||
);
|
||||
const body = captured.body as Array<Record<string, unknown>>;
|
||||
expect(body.map((item) => item.uuid)).toEqual(
|
||||
apps.map((app) => app.uid),
|
||||
);
|
||||
expect(body[2]!.associations).toEqual(['batchq']);
|
||||
expect(spies.getByName).not.toHaveBeenCalled();
|
||||
expect(spies.getByUid).not.toHaveBeenCalled();
|
||||
expect(spies.getFiletypeAssociations).not.toHaveBeenCalled();
|
||||
expect(spies.getByNames).toHaveBeenCalledTimes(1);
|
||||
expect(spies.getByUids).toHaveBeenCalledTimes(1);
|
||||
expect(spies.getFiletypeAssociationsByIds).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
for (const spy of Object.values(spies)) spy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /app-icon/:app_uid(/:size) ──────────────────────────────────
|
||||
@@ -1081,68 +897,6 @@ describe('AppController POST /rao actor gating', () => {
|
||||
).rejects.toMatchObject({ statusCode: 403, legacyCode: 'forbidden' });
|
||||
});
|
||||
|
||||
it('records an open for a full-access token — the account itself', async () => {
|
||||
const owner = await makeUser();
|
||||
const app = await createApp(owner.actor);
|
||||
const patActor = makeActor({
|
||||
user: owner.actor.user,
|
||||
accessToken: {
|
||||
uid: uuidv4(),
|
||||
issuer: owner.actor,
|
||||
authorized: null,
|
||||
fullAccess: true,
|
||||
},
|
||||
});
|
||||
|
||||
const { res, captured } = makeRes();
|
||||
await withActor(patActor, () =>
|
||||
callRoute(
|
||||
'post',
|
||||
'/rao',
|
||||
makeReq({ body: { app_uid: app.uid }, actor: patActor }),
|
||||
res,
|
||||
),
|
||||
);
|
||||
expect(captured.body).toEqual({});
|
||||
|
||||
await server.controllers.apps.drainPendingAppOpens();
|
||||
const rows = (await server.clients.db.read(
|
||||
'SELECT `app_uid` FROM `app_opens` WHERE `app_uid` = ? AND `user_id` = ?',
|
||||
[app.uid, owner.userId],
|
||||
)) as unknown[];
|
||||
expect(rows).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('refuses a full-access claim on an app-issued token', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const app = await createApp(actor);
|
||||
const issuer = makeActor({ ...actor, app: { uid: app.uid as string } });
|
||||
const tokenActor = makeActor({
|
||||
user: actor.user,
|
||||
accessToken: {
|
||||
uid: uuidv4(),
|
||||
issuer,
|
||||
authorized: null,
|
||||
fullAccess: true,
|
||||
},
|
||||
});
|
||||
|
||||
const { res } = makeRes();
|
||||
await expect(
|
||||
withActor(tokenActor, () =>
|
||||
callRoute(
|
||||
'post',
|
||||
'/rao',
|
||||
makeReq({
|
||||
body: { app_uid: app.uid },
|
||||
actor: tokenActor,
|
||||
}),
|
||||
res,
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 403, legacyCode: 'forbidden' });
|
||||
});
|
||||
|
||||
it("refuses to let an app actor record another app's open", async () => {
|
||||
const { actor } = await makeUser();
|
||||
const own = await createApp(actor);
|
||||
|
||||
@@ -17,17 +17,11 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { TOTP } from 'otpauth';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { makeActor } from '../../core/actor.js';
|
||||
import { runWithContext } from '../../core/context.js';
|
||||
import { createSecret as otpCreateSecret } from '../../services/auth/OTPUtil.js';
|
||||
import { FULL_API_ACCESS } from '../../services/permission/consts.js';
|
||||
import {
|
||||
createTestUser,
|
||||
setupPuterTestEnv,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
/**
|
||||
* HTTP-level coverage for `/auth/revoke-own-access-token` — the route that
|
||||
@@ -349,369 +343,4 @@ describe('revoke-own-access-token over HTTP', () => {
|
||||
}
|
||||
expect(await tokenReadStatus(file.uid, readToken)).not.toBe(200);
|
||||
});
|
||||
|
||||
// These three run last in the file's shared `env.users.user`: each one
|
||||
// revokes a session belonging to `owner.token` / `owner.workerToken`, so
|
||||
// later tests can't rely on those credentials still being live.
|
||||
|
||||
it('a revoked access token yields 401 with no reauth_token or auth_id', async () => {
|
||||
const owner = env.users.user;
|
||||
const file = await makeFile(owner);
|
||||
const readToken = await mintReadToken(owner.token, file.uid);
|
||||
|
||||
const revoke = await call(
|
||||
'POST',
|
||||
'/auth/revoke-own-access-token',
|
||||
owner.token,
|
||||
{ token: readToken },
|
||||
);
|
||||
expect(revoke.status).toBe(200);
|
||||
|
||||
const res = await call('GET', '/whoami', readToken);
|
||||
expect(res.status).toBe(401);
|
||||
const body = (await res.json()) as Record<string, unknown>;
|
||||
expect(body.code).toBe('reauth_required');
|
||||
expect(body.reauth_token).toBeUndefined();
|
||||
expect(body.auth_id).toBeUndefined();
|
||||
});
|
||||
|
||||
it('a revoked worker session gets no reauth_token even though it rides the session token type', async () => {
|
||||
const owner = env.users.user;
|
||||
const decoded = env.server.services.token.verify(
|
||||
'auth',
|
||||
owner.workerToken,
|
||||
) as { session_uid: string };
|
||||
await env.server.stores.session.removeByUuid(decoded.session_uid);
|
||||
|
||||
const res = await call('GET', '/whoami', owner.workerToken);
|
||||
expect(res.status).toBe(401);
|
||||
const body = (await res.json()) as Record<string, unknown>;
|
||||
expect(body.code).toBe('reauth_required');
|
||||
expect(body.reauth_token).toBeUndefined();
|
||||
expect(body.auth_id).toBeUndefined();
|
||||
});
|
||||
|
||||
it('a revoked GUI session still gets a reauth_token', async () => {
|
||||
const owner = env.users.user;
|
||||
const decoded = env.server.services.token.verify('auth', owner.token) as {
|
||||
session_uid: string;
|
||||
};
|
||||
await env.server.stores.session.removeByUuid(decoded.session_uid);
|
||||
|
||||
const res = await call('GET', '/whoami', owner.token);
|
||||
expect(res.status).toBe(401);
|
||||
const body = (await res.json()) as Record<string, unknown>;
|
||||
expect(body.code).toBe('reauth_required');
|
||||
expect(typeof body.reauth_token).toBe('string');
|
||||
|
||||
const user = await env.server.stores.user.getByUsername(
|
||||
owner.username,
|
||||
);
|
||||
expect(
|
||||
env.server.services.auth.verifyReauthToken(
|
||||
body.reauth_token as string,
|
||||
).authId,
|
||||
).toBe(user!.uuid);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Enrolling a second factor: setup sits behind the user-protected gate, and
|
||||
* enable needs a live code for the secret setup issued.
|
||||
*/
|
||||
describe('2FA enrollment over HTTP', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const PASSWORD = 'tfa-http-password';
|
||||
|
||||
const uniq = () => Math.random().toString(36).slice(2, 10);
|
||||
|
||||
/** A signed-in password account with a confirmed address. */
|
||||
const makeAccount = async () => {
|
||||
const username = `tfa_${uniq()}`;
|
||||
const { token } = await createTestUser(env.server, {
|
||||
username,
|
||||
password: PASSWORD,
|
||||
});
|
||||
const row = (await env.server.stores.user.getByUsername(username))!;
|
||||
await env.server.stores.user.update(row.id, {
|
||||
email: `${username}@test.local`,
|
||||
email_confirmed: 1,
|
||||
});
|
||||
await env.server.stores.user.invalidateById(row.id);
|
||||
return { username, token, userId: row.id };
|
||||
};
|
||||
|
||||
/** The user-protected gate takes the session cookie on the GUI origin. */
|
||||
const protectedPost = (
|
||||
path: string,
|
||||
token: string,
|
||||
body: Record<string, unknown>,
|
||||
credential: 'cookie' | 'bearer' = 'cookie',
|
||||
) =>
|
||||
fetch(new URL(path, env.origin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(credential === 'cookie'
|
||||
? { cookie: `puter_auth_token=${token}` }
|
||||
: { authorization: `Bearer ${token}` }),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const setup = (
|
||||
token: string,
|
||||
body: Record<string, unknown>,
|
||||
credential: 'cookie' | 'bearer' = 'cookie',
|
||||
) => protectedPost('/user-protected/setup-2fa', token, body, credential);
|
||||
|
||||
const configure = (
|
||||
token: string,
|
||||
action: string,
|
||||
body: Record<string, unknown> = {},
|
||||
) =>
|
||||
fetch(new URL(`/auth/configure-2fa/${action}`, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const otpState = async (userId: number) => {
|
||||
const row = await env.server.stores.user.getByProperty('id', userId, {
|
||||
force: true,
|
||||
});
|
||||
return {
|
||||
secret: row?.otp_secret ?? null,
|
||||
enabled: Boolean(row?.otp_enabled),
|
||||
};
|
||||
};
|
||||
|
||||
const liveTotp = (username: string, secret: string) =>
|
||||
new TOTP({
|
||||
issuer: 'puter.com',
|
||||
label: username,
|
||||
algorithm: 'SHA1',
|
||||
digits: 6,
|
||||
secret,
|
||||
}).generate();
|
||||
|
||||
it('refuses setup to a session that has not re-proved the password', async () => {
|
||||
const account = await makeAccount();
|
||||
|
||||
const noPassword = await setup(account.token, {});
|
||||
expect(noPassword.status).toBe(403);
|
||||
expect(await noPassword.text()).toContain('password_required');
|
||||
|
||||
const wrongPassword = await setup(account.token, { password: 'nope' });
|
||||
expect(wrongPassword.status).toBe(400);
|
||||
expect(await wrongPassword.text()).toContain('password_mismatch');
|
||||
|
||||
// Right password, but a bearer token is not the session cookie.
|
||||
const bearer = await setup(
|
||||
account.token,
|
||||
{ password: PASSWORD },
|
||||
'bearer',
|
||||
);
|
||||
expect(bearer.status).toBe(401);
|
||||
|
||||
// The old ungated path no longer hands out a secret.
|
||||
expect((await configure(account.token, 'setup')).status).toBe(400);
|
||||
|
||||
expect((await otpState(account.userId)).secret).toBeNull();
|
||||
});
|
||||
|
||||
it('enrolls after the password, and enables only on a live code', async () => {
|
||||
const account = await makeAccount();
|
||||
|
||||
const res = await setup(account.token, { password: PASSWORD });
|
||||
expect(res.status).toBe(200);
|
||||
const { secret, codes } = (await res.json()) as {
|
||||
secret: string;
|
||||
codes: string[];
|
||||
};
|
||||
expect(codes).toHaveLength(10);
|
||||
expect((await otpState(account.userId)).secret).toBe(secret);
|
||||
|
||||
expect((await configure(account.token, 'enable')).status).toBe(400);
|
||||
const foreign = otpCreateSecret(account.username).secret;
|
||||
const wrong = await configure(account.token, 'enable', {
|
||||
code: liveTotp(account.username, foreign),
|
||||
});
|
||||
expect(wrong.status).toBe(400);
|
||||
expect(await wrong.text()).toContain('code_mismatch');
|
||||
expect((await otpState(account.userId)).enabled).toBe(false);
|
||||
|
||||
const enabled = await configure(account.token, 'enable', {
|
||||
code: liveTotp(account.username, secret),
|
||||
});
|
||||
expect(enabled.status).toBe(200);
|
||||
expect((await otpState(account.userId)).enabled).toBe(true);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
/**
|
||||
* A temp account re-attaches through `/signup` only from a session that ran
|
||||
* out, not from one that was revoked.
|
||||
*/
|
||||
describe('temp-account reauth over HTTP', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
}, 120_000);
|
||||
|
||||
const signup = (body: Record<string, unknown>) =>
|
||||
fetch(new URL('/signup', env.origin), {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
it('a revoked temp session cannot be revived with its reauth_token', async () => {
|
||||
const first = await signup({ is_temp: true });
|
||||
expect(first.status).toBe(200);
|
||||
const { token } = (await first.json()) as { token: string };
|
||||
|
||||
const decoded = env.server.services.token.verify('auth', token) as {
|
||||
session_uid: string;
|
||||
};
|
||||
await env.server.services.auth.revokeSession(decoded.session_uid);
|
||||
|
||||
const who = await fetch(new URL('/whoami', env.apiOrigin), {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
expect(who.status).toBe(401);
|
||||
const { reauth_token } = (await who.json()) as {
|
||||
reauth_token?: string;
|
||||
};
|
||||
expect(typeof reauth_token).toBe('string');
|
||||
|
||||
const revive = await signup({ is_temp: true, reauth_token });
|
||||
expect(revive.status).toBe(401);
|
||||
});
|
||||
|
||||
it('an expired temp session re-attaches to the same account', async () => {
|
||||
const first = await signup({ is_temp: true });
|
||||
expect(first.status).toBe(200);
|
||||
const { user } = (await first.json()) as { user: { uuid: string } };
|
||||
|
||||
const reauth_token = env.server.services.auth.signReauthToken(
|
||||
user.uuid,
|
||||
'session_expired',
|
||||
);
|
||||
const again = await signup({ is_temp: true, reauth_token });
|
||||
expect(again.status).toBe(200);
|
||||
const body = (await again.json()) as {
|
||||
token: string;
|
||||
user: { uuid: string };
|
||||
};
|
||||
expect(body.user.uuid).toBe(user.uuid);
|
||||
const who = await fetch(new URL('/whoami', env.apiOrigin), {
|
||||
headers: { authorization: `Bearer ${body.token}` },
|
||||
});
|
||||
expect(who.status).toBe(200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('session token helpers over HTTP', () => {
|
||||
let env: PuterTestEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
}, 120_000);
|
||||
|
||||
const signupTemp = async () => {
|
||||
const res = await fetch(new URL('/signup', env.origin), {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ is_temp: true }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
return (await res.json()) as { token: string; user: { uuid: string } };
|
||||
};
|
||||
|
||||
const get = (path: string, token: string) =>
|
||||
fetch(new URL(path, env.origin), {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
const claims = (token: string) =>
|
||||
env.server.services.token.verify('auth', token) as {
|
||||
type: string;
|
||||
user_uid: string;
|
||||
session_uid: string;
|
||||
};
|
||||
|
||||
it('mint tokens for the session the request authenticated with', async () => {
|
||||
const { token, user } = await signupTemp();
|
||||
const session = claims(token);
|
||||
const row = await env.server.stores.user.getByUuid(user.uuid);
|
||||
const getById = vi.spyOn(env.server.stores.user, 'getById');
|
||||
try {
|
||||
const gui = await get('/get-gui-token', token);
|
||||
expect(gui.status).toBe(200);
|
||||
const { token: guiToken } = (await gui.json()) as {
|
||||
token: string;
|
||||
};
|
||||
expect(claims(guiToken)).toMatchObject({
|
||||
type: 'gui',
|
||||
user_uid: user.uuid,
|
||||
session_uid: session.session_uid,
|
||||
});
|
||||
|
||||
const sync = await get('/session/sync-cookie', token);
|
||||
expect(sync.status).toBe(204);
|
||||
const cookie = sync.headers.get('set-cookie') ?? '';
|
||||
const cookieToken = /puter_auth_token=([^;]+)/.exec(cookie)?.[1];
|
||||
expect(claims(cookieToken!)).toMatchObject({
|
||||
type: 'session',
|
||||
user_uid: user.uuid,
|
||||
session_uid: session.session_uid,
|
||||
});
|
||||
|
||||
// The user is the row token auth already read.
|
||||
expect(getById.mock.calls.filter(([id]) => id === row!.id)).toEqual(
|
||||
[],
|
||||
);
|
||||
} finally {
|
||||
getById.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("refuse a deleted account's session before either handler runs", async () => {
|
||||
const { token, user } = await signupTemp();
|
||||
const row = await env.server.stores.user.getByUuid(user.uuid);
|
||||
await env.server.clients.db.write('DELETE FROM `user` WHERE `id` = ?', [
|
||||
row!.id,
|
||||
]);
|
||||
await env.server.stores.user.markDeleted(row!);
|
||||
|
||||
expect((await get('/get-gui-token', token)).status).toBe(401);
|
||||
const sync = await get('/session/sync-cookie', token);
|
||||
expect(sync.status).toBe(401);
|
||||
expect(sync.headers.get('set-cookie') ?? '').not.toContain(
|
||||
'puter_auth_token=',
|
||||
);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,371 +0,0 @@
|
||||
import { TOTP } from 'otpauth';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
createTestUser,
|
||||
setupPuterTestEnv,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
import { installSeatStub } from '../../testFixtures/seatStub.js';
|
||||
|
||||
describe('account change second-factor verification', () => {
|
||||
let env: PuterTestEnv;
|
||||
const password = 'email-change-password';
|
||||
const links = new Map<string, string>();
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
vi.spyOn(env.server.clients.email, 'send').mockImplementation(
|
||||
async (to, template, values) => {
|
||||
if (template === 'email_verification_link')
|
||||
links.set(to, String(values?.link));
|
||||
return null;
|
||||
},
|
||||
);
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const account = async (enabled = true, oidcOnly = false) => {
|
||||
const username = `email2fa_${uuidv4().slice(0, 8)}`;
|
||||
const credentials = await createTestUser(env.server, {
|
||||
username,
|
||||
password,
|
||||
});
|
||||
const user = (await env.server.stores.user.getByUsername(username))!;
|
||||
const otp = new TOTP({
|
||||
issuer: 'puter.com',
|
||||
label: username,
|
||||
algorithm: 'SHA1',
|
||||
digits: 6,
|
||||
});
|
||||
await env.server.stores.user.update(user.id, {
|
||||
email: `${username}@example.com`,
|
||||
email_confirmed: true,
|
||||
otp_enabled: enabled,
|
||||
otp_secret: otp.secret.base32,
|
||||
...(oidcOnly ? { password: null } : {}),
|
||||
});
|
||||
return { ...credentials, id: user.id, uuid: user.uuid, otp };
|
||||
};
|
||||
|
||||
const change = (
|
||||
token: string,
|
||||
body: Record<string, unknown>,
|
||||
revalidation?: string,
|
||||
path = '/user-protected/change-email',
|
||||
) =>
|
||||
fetch(new URL(path, env.origin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
cookie: `puter_auth_token=${token}${revalidation ? `; puter_revalidation=${revalidation}` : ''}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
it.each([undefined, 'invalid-code', 123456, {}])(
|
||||
'rejects an enabled account with code %j before staging a change',
|
||||
async (code) => {
|
||||
const user = await account();
|
||||
const response = await change(user.token, {
|
||||
password,
|
||||
new_email: '[email protected]',
|
||||
code,
|
||||
});
|
||||
expect(response.status).toBeGreaterThanOrEqual(400);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
expect(await response.json()).toMatchObject({ factor: 'otp' });
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.change_email_confirm_token).toBeFalsy();
|
||||
expect(after!.unconfirmed_change_email).toBeFalsy();
|
||||
},
|
||||
);
|
||||
|
||||
it('rejects an incorrect six-digit code without staging the address', async () => {
|
||||
const user = await account();
|
||||
const code = ['000000', '000001', '000002', '000003'].find(
|
||||
(token) => user.otp.validate({ token, window: 1 }) === null,
|
||||
);
|
||||
const response = await change(user.token, {
|
||||
password,
|
||||
new_email: '[email protected]',
|
||||
code,
|
||||
});
|
||||
expect(response.status).toBe(401);
|
||||
expect(await response.json()).toMatchObject({
|
||||
code: 'code_mismatch',
|
||||
factor: 'otp',
|
||||
});
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, { force: true }))!
|
||||
.change_email_confirm_token,
|
||||
).toBeFalsy();
|
||||
});
|
||||
|
||||
it('refuses a change when the enabled account secret is unavailable', async () => {
|
||||
const user = await account();
|
||||
await env.server.stores.user.update(user.id, {
|
||||
otp_secret: 'invalid!',
|
||||
});
|
||||
expect(
|
||||
(
|
||||
await change(user.token, {
|
||||
password,
|
||||
new_email: '[email protected]',
|
||||
code: '000000',
|
||||
})
|
||||
).status,
|
||||
).toBe(401);
|
||||
});
|
||||
|
||||
it('requires a code even after OIDC revalidation', async () => {
|
||||
const user = await account(true, true);
|
||||
const proof = env.server.services.token.sign(
|
||||
'oidc-state',
|
||||
{
|
||||
user_uuid: user.uuid,
|
||||
purpose: 'revalidate',
|
||||
},
|
||||
{ expiresIn: '5m' },
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await change(
|
||||
user.token,
|
||||
{ new_email: '[email protected]' },
|
||||
proof,
|
||||
)
|
||||
).status,
|
||||
).toBe(400);
|
||||
expect(
|
||||
(
|
||||
await change(
|
||||
user.token,
|
||||
{
|
||||
new_email: '[email protected]',
|
||||
code: user.otp.generate(),
|
||||
},
|
||||
proof,
|
||||
)
|
||||
).status,
|
||||
).toBe(200);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'completes the change with the required factors (2FA enabled: %s)',
|
||||
async (enabled) => {
|
||||
const user = await account(enabled);
|
||||
const newEmail = `${uuidv4()}@example.com`;
|
||||
expect(
|
||||
(
|
||||
await change(user.token, {
|
||||
password,
|
||||
new_email: newEmail,
|
||||
...(enabled ? { code: user.otp.generate() } : {}),
|
||||
})
|
||||
).status,
|
||||
).toBe(200);
|
||||
const link = links.get(newEmail)!;
|
||||
expect(link).toBeTruthy();
|
||||
expect((await fetch(link, { redirect: 'manual' })).status).toBe(
|
||||
200,
|
||||
);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
}))!.email,
|
||||
).toBe(newEmail);
|
||||
},
|
||||
);
|
||||
|
||||
it('requires a new request if 2FA is enabled after the link was issued', async () => {
|
||||
const user = await account(false);
|
||||
const newEmail = `${uuidv4()}@example.com`;
|
||||
expect(
|
||||
(await change(user.token, { password, new_email: newEmail }))
|
||||
.status,
|
||||
).toBe(200);
|
||||
await env.server.stores.user.update(user.id, { otp_enabled: true });
|
||||
expect(
|
||||
(await fetch(links.get(newEmail)!, { redirect: 'manual' })).status,
|
||||
).toBe(400);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, { force: true }))!
|
||||
.email,
|
||||
).not.toBe(newEmail);
|
||||
});
|
||||
|
||||
it('refuses an existing confirmation link that never verified 2FA', async () => {
|
||||
const user = await account();
|
||||
const token = uuidv4();
|
||||
await env.server.stores.user.update(user.id, {
|
||||
change_email_confirm_token: token,
|
||||
unconfirmed_change_email: '[email protected]',
|
||||
});
|
||||
const link = env.server.services.token.sign(
|
||||
'otp',
|
||||
{
|
||||
token,
|
||||
user_id: user.id,
|
||||
purpose: 'change-email',
|
||||
},
|
||||
{ expiresIn: '1h' },
|
||||
);
|
||||
const response = await fetch(
|
||||
new URL(`/change_email/confirm?token=${link}`, env.origin),
|
||||
{ redirect: 'manual' },
|
||||
);
|
||||
expect(response.status).toBe(400);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, { force: true }))!
|
||||
.email,
|
||||
).toBe(`${user.username}@example.com`);
|
||||
});
|
||||
describe('disabling 2FA', () => {
|
||||
it.each([undefined, 'invalid-code', 123456, {}])(
|
||||
'refuses code %j and retains the second-factor configuration',
|
||||
async (code) => {
|
||||
const user = await account();
|
||||
await env.server.stores.user.update(user.id, {
|
||||
otp_recovery_codes: 'one,two,three',
|
||||
});
|
||||
const response = await change(
|
||||
user.token,
|
||||
{ password, code },
|
||||
undefined,
|
||||
'/user-protected/disable-2fa',
|
||||
);
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.json()).toMatchObject({ factor: 'otp' });
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.otp_enabled).toBeTruthy();
|
||||
expect(after!.otp_secret).toBe(user.otp.secret.base32);
|
||||
expect(after!.otp_recovery_codes).toBe('one,two,three');
|
||||
},
|
||||
);
|
||||
|
||||
it('rejects an incorrect six-digit code', async () => {
|
||||
const user = await account();
|
||||
const code = ['000000', '000001', '000002', '000003'].find(
|
||||
(token) => user.otp.validate({ token, window: 1 }) === null,
|
||||
);
|
||||
const response = await change(
|
||||
user.token,
|
||||
{ password, code },
|
||||
undefined,
|
||||
'/user-protected/disable-2fa',
|
||||
);
|
||||
expect(response.status).toBe(401);
|
||||
expect(await response.json()).toMatchObject({
|
||||
code: 'code_mismatch',
|
||||
factor: 'otp',
|
||||
});
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
}))!.otp_enabled,
|
||||
).toBeTruthy();
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'accepts a current code after proving identity (OIDC-only: %s)',
|
||||
async (oidcOnly) => {
|
||||
const user = await account(true, oidcOnly);
|
||||
const proof = oidcOnly
|
||||
? env.server.services.token.sign(
|
||||
'oidc-state',
|
||||
{
|
||||
user_uuid: user.uuid,
|
||||
purpose: 'revalidate',
|
||||
},
|
||||
{ expiresIn: '5m' },
|
||||
)
|
||||
: undefined;
|
||||
const missing = await change(
|
||||
user.token,
|
||||
oidcOnly ? {} : { password },
|
||||
proof,
|
||||
'/user-protected/disable-2fa',
|
||||
);
|
||||
expect(missing.status).toBe(400);
|
||||
const response = await change(
|
||||
user.token,
|
||||
{
|
||||
...(oidcOnly ? {} : { password }),
|
||||
code: user.otp.generate(),
|
||||
},
|
||||
proof,
|
||||
'/user-protected/disable-2fa',
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.otp_enabled).toBeFalsy();
|
||||
expect(after!.otp_secret).toBeNull();
|
||||
expect(after!.otp_recovery_codes).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it('does not accept a code without password or OIDC revalidation', async () => {
|
||||
const user = await account();
|
||||
const response = await change(
|
||||
user.token,
|
||||
{ code: user.otp.generate() },
|
||||
undefined,
|
||||
'/user-protected/disable-2fa',
|
||||
);
|
||||
expect(response.status).toBe(403);
|
||||
expect(await response.json()).toMatchObject({
|
||||
code: 'password_required',
|
||||
});
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
}))!.otp_enabled,
|
||||
).toBeTruthy();
|
||||
});
|
||||
|
||||
it('does not let a valid code override a team requirement for 2FA', async () => {
|
||||
const member = await account(true, true);
|
||||
installSeatStub(env.server as never, [member.id], {
|
||||
require2fa: true,
|
||||
});
|
||||
const proof = env.server.services.token.sign('oidc-state', {
|
||||
user_uuid: member.uuid, purpose: 'revalidate',
|
||||
}, { expiresIn: '5m' });
|
||||
const response = await change(
|
||||
member.token,
|
||||
{ code: member.otp.generate() },
|
||||
proof,
|
||||
'/user-protected/disable-2fa',
|
||||
);
|
||||
expect(response.status).toBe(409);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(member.id, {
|
||||
force: true,
|
||||
}))!.otp_enabled,
|
||||
).toBeTruthy();
|
||||
});
|
||||
|
||||
it('remains idempotent for an account with 2FA already disabled', async () => {
|
||||
const user = await account(false);
|
||||
expect(
|
||||
(
|
||||
await change(
|
||||
user.token,
|
||||
{ password },
|
||||
undefined,
|
||||
'/user-protected/disable-2fa',
|
||||
)
|
||||
).status,
|
||||
).toBe(200);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,426 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { isAccountContext } from '../../core/actor.js';
|
||||
import type { AccessTokenPayload } from '../../services/auth/types.js';
|
||||
import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
|
||||
|
||||
// Seeded with godmode on.
|
||||
const DEV_CENTER = 'app-0b37f054-07d4-4627-8765-11bd23e889d4';
|
||||
const TWELVE_HOURS = 12 * 60 * 60;
|
||||
|
||||
/**
|
||||
* A godmode app launched from the desktop runs on a full-access token of its
|
||||
* own: tied to the desktop session that asked for it, carrying the app for
|
||||
* attribution, and expiring unless the desktop asks again.
|
||||
*/
|
||||
describe('godmode app tokens', () => {
|
||||
let env: PuterTestEnv;
|
||||
let userId = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
const row = await env.server.stores.user.getByUsername(
|
||||
env.users.user.username,
|
||||
);
|
||||
userId = row!.id;
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
|
||||
const call = (
|
||||
method: string,
|
||||
path: string,
|
||||
token: string,
|
||||
body?: unknown,
|
||||
) =>
|
||||
fetch(new URL(path, env.apiOrigin), {
|
||||
method,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${token}`,
|
||||
},
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||
});
|
||||
|
||||
const mint = async (sessionToken: string, appUid = DEV_CENTER) => {
|
||||
const res = await call(
|
||||
'POST',
|
||||
'/auth/get-user-app-token',
|
||||
sessionToken,
|
||||
{ app_uid: appUid },
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
return (await res.json()) as {
|
||||
token: string | null;
|
||||
app_uid: string;
|
||||
godmode?: boolean;
|
||||
expires_at?: number;
|
||||
};
|
||||
};
|
||||
|
||||
const decode = (token: string) =>
|
||||
env.server.services.token.verify<AccessTokenPayload>('auth', token);
|
||||
|
||||
const live = async (token: string) =>
|
||||
(await env.server.services.auth.authenticate(token)).actor !== undefined;
|
||||
|
||||
/** A read token minted the way `getReadURL()` does, by `minter`. */
|
||||
const mintScoped = async (minter: string) => {
|
||||
const res = await call('POST', '/auth/create-access-token', minter, {
|
||||
permissions: ['service:foo:ii:read'],
|
||||
expiresIn: '7d',
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
return ((await res.json()) as { token: string }).token;
|
||||
};
|
||||
|
||||
/** Rewrite a session row in place, as time passing would. */
|
||||
const setRow = async (uuid: string, fields: Record<string, number>) => {
|
||||
const cols = Object.keys(fields);
|
||||
await env.server.clients.db.write(
|
||||
`UPDATE \`sessions\` SET ${cols.map((c) => `\`${c}\` = ?`).join(', ')} WHERE \`uuid\` = ?`,
|
||||
[...cols.map((c) => fields[c]), uuid],
|
||||
);
|
||||
await env.server.stores.session.publishCacheKeys({
|
||||
keys: [`sessions:v2:uuid:${uuid}`],
|
||||
});
|
||||
};
|
||||
|
||||
/** A desktop session of its own, so revoking it leaves the shared one up. */
|
||||
const newDesktopSession = async () => {
|
||||
const user = await env.server.stores.user.getById(userId);
|
||||
return env.server.services.auth.createSessionToken(user!);
|
||||
};
|
||||
|
||||
/** An app row of its own, so demoting it disturbs nothing else. */
|
||||
const makeApp = async ({
|
||||
name = `gm-${crypto.randomUUID().slice(0, 8)}`,
|
||||
godmode = true,
|
||||
} = {}) => {
|
||||
const uid = `app-${crypto.randomUUID()}`;
|
||||
await env.server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `owner_user_id`, `name`, `title`, `index_url`, `godmode`, `timestamp`) VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)',
|
||||
[uid, userId, name, 'Test App', `https://${name}.test/`, godmode ? 1 : 0],
|
||||
);
|
||||
return { uid, name };
|
||||
};
|
||||
|
||||
it('issues a full-access token that carries the app', async () => {
|
||||
const before = Math.floor(Date.now() / 1000);
|
||||
const body = await mint(env.users.user.token);
|
||||
|
||||
expect(body).toMatchObject({ app_uid: DEV_CENTER, godmode: true });
|
||||
expect(body.expires_at).toBeGreaterThanOrEqual(before + TWELVE_HOURS);
|
||||
|
||||
const claims = decode(body.token!);
|
||||
expect(claims).toMatchObject({
|
||||
type: 'access-token',
|
||||
full_access: true,
|
||||
godmode_app_uid: DEV_CENTER,
|
||||
});
|
||||
expect(claims.app_uid).toBeUndefined();
|
||||
// The row's expiry is what lapses; the JWT itself never does.
|
||||
expect(claims.exp).toBeUndefined();
|
||||
|
||||
const { actor } = await env.server.services.auth.authenticate(
|
||||
body.token!,
|
||||
);
|
||||
expect(actor).toBeDefined();
|
||||
expect(isAccountContext(actor!)).toBe(true);
|
||||
expect(actor!.effectiveApp).toBeNull();
|
||||
expect(actor!.accessToken).toMatchObject({
|
||||
fullAccess: true,
|
||||
godmodeApp: { uid: DEV_CENTER },
|
||||
});
|
||||
});
|
||||
|
||||
it('reaches the account surface but not account management', async () => {
|
||||
const { token } = await mint(env.users.user.token);
|
||||
|
||||
expect((await call('GET', '/whoami', token!)).status).toBe(200);
|
||||
expect((await call('GET', '/get-dev-profile', token!)).status).toBe(
|
||||
200,
|
||||
);
|
||||
expect((await call('GET', '/auth/list-sessions', token!)).status).toBe(
|
||||
403,
|
||||
);
|
||||
// It can't mint its own replacement; only the desktop can.
|
||||
const self = await call('POST', '/auth/get-user-app-token', token!, {
|
||||
app_uid: DEV_CENTER,
|
||||
});
|
||||
expect(self.status).toBe(403);
|
||||
});
|
||||
|
||||
it('keeps one token per desktop session across renewals', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const first = await mint(desktop.gui_token);
|
||||
const second = await mint(desktop.gui_token);
|
||||
|
||||
expect(decode(second.token!).token_uid).toBe(
|
||||
decode(first.token!).token_uid,
|
||||
);
|
||||
// Asked again within the hour: the row keeps the expiry it has.
|
||||
expect(second.expires_at).toBe(first.expires_at);
|
||||
// A copy taken at launch keeps working after the desktop renews.
|
||||
expect(await live(first.token!)).toBe(true);
|
||||
|
||||
const row = await env.server.stores.session.getByUuidAny(
|
||||
decode(second.token!).session_uid!,
|
||||
);
|
||||
expect(row).toMatchObject({
|
||||
kind: 'access_token',
|
||||
app_uid: DEV_CENTER,
|
||||
expires_at: second.expires_at,
|
||||
});
|
||||
});
|
||||
|
||||
it('is extended once the desktop asks with under eleven hours left', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const first = await mint(desktop.gui_token);
|
||||
const sessionUid = decode(first.token!).session_uid!;
|
||||
await setRow(sessionUid, {
|
||||
expires_at: Math.floor(Date.now() / 1000) + 2 * 60 * 60,
|
||||
});
|
||||
|
||||
const before = Math.floor(Date.now() / 1000);
|
||||
const renewed = await mint(desktop.gui_token);
|
||||
expect(decode(renewed.token!).token_uid).toBe(
|
||||
decode(first.token!).token_uid,
|
||||
);
|
||||
expect(renewed.expires_at).toBeGreaterThanOrEqual(
|
||||
before + TWELVE_HOURS,
|
||||
);
|
||||
const row = await env.server.stores.session.getByUuidAny(sessionUid);
|
||||
expect(row?.expires_at).toBe(renewed.expires_at);
|
||||
});
|
||||
|
||||
it('is never extended by being used', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const { token, expires_at } = await mint(desktop.gui_token);
|
||||
expect((await call('GET', '/whoami', token!)).status).toBe(200);
|
||||
const row = await env.server.stores.session.getByUuidAny(
|
||||
decode(token!).session_uid!,
|
||||
);
|
||||
expect(row?.expires_at).toBe(expires_at);
|
||||
});
|
||||
|
||||
it('lists the token under its app for the account', async () => {
|
||||
await mint(env.users.user.token);
|
||||
const res = await call(
|
||||
'GET',
|
||||
'/auth/list-sessions',
|
||||
env.users.user.token,
|
||||
);
|
||||
const sessions = (await res.json()) as Array<{
|
||||
kind: string;
|
||||
app_uid: string | null;
|
||||
app: { name?: string } | null;
|
||||
}>;
|
||||
const row = sessions.find(
|
||||
(s) => s.kind === 'access_token' && s.app_uid === DEV_CENTER,
|
||||
);
|
||||
expect(row?.app?.name).toBe('dev-center');
|
||||
});
|
||||
|
||||
it('installs the app like any other launch, and uninstalls the same way', async () => {
|
||||
const { uid } = await makeApp();
|
||||
const app = await env.server.stores.app.getByUid(uid);
|
||||
const installed = () =>
|
||||
env.server.stores.permission.hasUserAppPerm(
|
||||
userId,
|
||||
app!.id,
|
||||
'flag:app-is-authenticated',
|
||||
);
|
||||
|
||||
const { token } = await mint(env.users.user.token, uid);
|
||||
expect(await installed()).toBe(true);
|
||||
expect(
|
||||
await env.server.stores.fsEntry.getEntryByPath(
|
||||
`/${env.users.user.username}/AppData/${uid}`,
|
||||
),
|
||||
).toBeTruthy();
|
||||
|
||||
// What the desktop's uninstall does: drop the grants, then the app's sessions.
|
||||
const revoked = await call(
|
||||
'POST',
|
||||
'/auth/revoke-user-app',
|
||||
env.users.user.token,
|
||||
{ app_uid: uid, permission: '*' },
|
||||
);
|
||||
expect(revoked.status).toBe(200);
|
||||
expect(await installed()).toBe(false);
|
||||
|
||||
await env.server.services.auth.revokeSession(decode(token!).session_uid!);
|
||||
expect(await live(token!)).toBe(false);
|
||||
});
|
||||
|
||||
it('dies with the desktop session that minted it', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const { token } = await mint(desktop.gui_token);
|
||||
expect((await call('GET', '/whoami', token!)).status).toBe(200);
|
||||
|
||||
await env.server.services.auth.revokeSession(
|
||||
desktop.session.uuid as string,
|
||||
);
|
||||
|
||||
const res = await call('GET', '/whoami', token!);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('stops working once its app is no longer godmode', async () => {
|
||||
const app = await makeApp();
|
||||
const { token } = await mint(env.users.user.token, app.uid);
|
||||
expect((await call('GET', '/whoami', token!)).status).toBe(200);
|
||||
|
||||
const row = await env.server.stores.app.getByUid(app.uid);
|
||||
await env.server.clients.db.write(
|
||||
'UPDATE `apps` SET `godmode` = 0 WHERE `uid` = ?',
|
||||
[app.uid],
|
||||
);
|
||||
await env.server.stores.app.invalidate(row);
|
||||
|
||||
expect((await call('GET', '/whoami', token!)).status).toBe(401);
|
||||
});
|
||||
|
||||
it('still gives an ordinary app an ordinary app token', async () => {
|
||||
const app = await makeApp({ godmode: false });
|
||||
const body = await mint(env.users.user.token, app.uid);
|
||||
expect(body.godmode).toBeUndefined();
|
||||
expect(decode(body.token!).type).toBe('app-under-user');
|
||||
});
|
||||
|
||||
it('mints scoped tokens that are revoked along with it', async () => {
|
||||
const { token } = await mint(env.users.user.token);
|
||||
const child = await call(
|
||||
'POST',
|
||||
'/auth/create-access-token',
|
||||
token!,
|
||||
{ permissions: ['service:foo:ii:read'], expiresIn: '1h' },
|
||||
);
|
||||
expect(child.status).toBe(200);
|
||||
const childToken = ((await child.json()) as { token: string }).token;
|
||||
expect(
|
||||
(await env.server.services.auth.authenticate(childToken)).actor,
|
||||
).toBeDefined();
|
||||
|
||||
const full = await call('POST', '/auth/create-access-token', token!, {
|
||||
permissions: ['full-api-access'],
|
||||
});
|
||||
expect(full.status).toBe(403);
|
||||
|
||||
const { actor } = await env.server.services.auth.authenticate(
|
||||
env.users.user.token,
|
||||
);
|
||||
await env.server.services.auth.revokeAccessToken(actor!, token!);
|
||||
|
||||
expect(
|
||||
(await env.server.services.auth.authenticate(childToken)).actor,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('stays dead once it lapses; the desktop asking again gets a new one', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const { token } = await mint(desktop.gui_token);
|
||||
const child = await mintScoped(token!);
|
||||
expect(await live(child)).toBe(true);
|
||||
|
||||
// The desktop stopped renewing it: its row's expiry passes.
|
||||
await setRow(decode(token!).session_uid!, {
|
||||
expires_at: Math.floor(Date.now() / 1000) - 1,
|
||||
});
|
||||
expect(await live(token!)).toBe(false);
|
||||
expect(await live(child)).toBe(false);
|
||||
|
||||
const next = await mint(desktop.gui_token);
|
||||
expect(decode(next.token!).token_uid).not.toBe(decode(token!).token_uid);
|
||||
expect(await live(next.token!)).toBe(true);
|
||||
expect(await live(token!)).toBe(false);
|
||||
expect(await live(child)).toBe(false);
|
||||
});
|
||||
|
||||
it('is replaced and revoked once past its max age, however often renewed', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const { token } = await mint(desktop.gui_token);
|
||||
const child = await mintScoped(token!);
|
||||
await setRow(decode(token!).session_uid!, {
|
||||
created_at: Math.floor(Date.now() / 1000) - 8 * 24 * 60 * 60,
|
||||
});
|
||||
|
||||
const next = await mint(desktop.gui_token);
|
||||
expect(decode(next.token!).token_uid).not.toBe(decode(token!).token_uid);
|
||||
expect(await live(next.token!)).toBe(true);
|
||||
expect(await live(token!)).toBe(false);
|
||||
expect(await live(child)).toBe(false);
|
||||
});
|
||||
|
||||
it('lets the tokens it minted die with the desktop session', async () => {
|
||||
const desktop = await newDesktopSession();
|
||||
const { token } = await mint(desktop.gui_token);
|
||||
const child = await mintScoped(token!);
|
||||
expect(await live(child)).toBe(true);
|
||||
|
||||
await env.server.services.auth.revokeSession(
|
||||
desktop.session.uuid as string,
|
||||
);
|
||||
|
||||
expect(await live(token!)).toBe(false);
|
||||
expect(await live(child)).toBe(false);
|
||||
});
|
||||
|
||||
it('reaches the AI wire routes, where a gate listener may waive the plan', async () => {
|
||||
const { token } = await mint(env.users.user.token);
|
||||
const route = '/puterai/openai/v1/chat/completions';
|
||||
const send = (bearer: string) =>
|
||||
fetch(new URL(route, env.apiOrigin), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${bearer}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'fake',
|
||||
provider: 'fake-chat',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
max_tokens: 16,
|
||||
}),
|
||||
});
|
||||
|
||||
// Past the credential gates, stopped only by the plan.
|
||||
expect((await send(token!)).status).toBe(402);
|
||||
|
||||
env.server.clients.event.on(
|
||||
'subscription.gate.route.post.puterai.openai.v1.chat.completions',
|
||||
(_key, event) => {
|
||||
const gate = event as {
|
||||
actor: { accessToken?: { godmodeApp?: { uid: string } } };
|
||||
allow: boolean;
|
||||
};
|
||||
if (gate.actor.accessToken?.godmodeApp?.uid === DEV_CENTER) {
|
||||
gate.allow = true;
|
||||
}
|
||||
},
|
||||
);
|
||||
expect((await send(token!)).status).not.toBe(402);
|
||||
expect((await send(env.users.user.apiToken)).status).toBe(402);
|
||||
});
|
||||
});
|
||||
@@ -1,96 +0,0 @@
|
||||
import crypto from 'node:crypto';
|
||||
import type { Response } from 'express';
|
||||
import { HttpError } from '../../core/http/HttpError.js';
|
||||
import type { OIDCService } from '../../services/auth/OIDCService.js';
|
||||
import type { UserRow } from '../../stores/user/UserStore.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
import { sessionCookieFlags } from '../../util/cookieFlags.js';
|
||||
|
||||
export const OIDC_POPUP_RETURN_COOKIE_NAME = 'puter_oidc_popup_return';
|
||||
export const OIDC_POPUP_RETURN_EXPIRY_SEC = 300;
|
||||
|
||||
export interface OidcLoginContinuation {
|
||||
userUuid: string;
|
||||
redirectUri: string;
|
||||
embeddedInPopup: boolean;
|
||||
openerOrigin: string | null;
|
||||
messageId: string | null;
|
||||
switchedToLogin: boolean;
|
||||
}
|
||||
|
||||
interface OidcReturnDeps {
|
||||
config: IConfig;
|
||||
oidc: OIDCService;
|
||||
}
|
||||
|
||||
/** Bind an attested popup return to the browser that completed sign-in. */
|
||||
export function mintOidcPopupReturn(
|
||||
res: Response,
|
||||
payload: Record<string, unknown>,
|
||||
deps: OidcReturnDeps,
|
||||
): string {
|
||||
const nonce = crypto.randomBytes(32).toString('base64url');
|
||||
res.cookie(OIDC_POPUP_RETURN_COOKIE_NAME, nonce, {
|
||||
...sessionCookieFlags(deps.config, { crossSite: false }),
|
||||
httpOnly: true,
|
||||
maxAge: OIDC_POPUP_RETURN_EXPIRY_SEC * 1000,
|
||||
path: '/',
|
||||
});
|
||||
return deps.oidc.signPopupReturn({ action: 'sign-in', ...payload, nonce });
|
||||
}
|
||||
|
||||
/** Call only after both the provider and the account's second factor succeed. */
|
||||
export function completeOidcLoginReturn(
|
||||
res: Response,
|
||||
user: Pick<UserRow, 'uuid'>,
|
||||
continuation: unknown,
|
||||
deps: OidcReturnDeps,
|
||||
): string {
|
||||
if (!continuation || typeof continuation !== 'object') {
|
||||
throw new HttpError(400, 'Invalid login continuation.', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
const data = continuation as OidcLoginContinuation;
|
||||
if (
|
||||
data.userUuid !== user.uuid ||
|
||||
typeof data.redirectUri !== 'string' ||
|
||||
typeof data.embeddedInPopup !== 'boolean' ||
|
||||
typeof data.switchedToLogin !== 'boolean' ||
|
||||
(data.openerOrigin !== null && typeof data.openerOrigin !== 'string') ||
|
||||
(data.messageId !== null && typeof data.messageId !== 'string')
|
||||
) {
|
||||
throw new HttpError(400, 'Invalid login continuation.', {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
const origin = (deps.config.origin ?? '').replace(/\/$/, '');
|
||||
let target = data.redirectUri || origin || '/';
|
||||
try {
|
||||
if (origin && new URL(target).origin !== new URL(origin).origin)
|
||||
target = origin;
|
||||
} catch {
|
||||
target = origin || '/';
|
||||
}
|
||||
const append = (key: string, value: string) => {
|
||||
target += `${target.includes('?') ? '&' : '?'}${encodeURIComponent(key)}=${encodeURIComponent(value)}`;
|
||||
};
|
||||
if (data.embeddedInPopup) {
|
||||
append('oidc_login', 'true');
|
||||
append(
|
||||
'opener_state',
|
||||
mintOidcPopupReturn(
|
||||
res,
|
||||
{
|
||||
opener_origin: data.openerOrigin,
|
||||
msg_id: data.messageId,
|
||||
oidc_login: true,
|
||||
user_uuid: user.uuid,
|
||||
},
|
||||
deps,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (data.switchedToLogin) append('oidc_switched', 'login');
|
||||
return target;
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { hashRecoveryCode } from '../../services/auth/OTPUtil.js';
|
||||
import {
|
||||
createTestUser,
|
||||
setupPuterTestEnv,
|
||||
type PuterTestEnv,
|
||||
} from '../../testUtil.js';
|
||||
|
||||
describe('recovery code consumption', () => {
|
||||
let env: PuterTestEnv;
|
||||
const password = 'recovery-code-password';
|
||||
|
||||
beforeAll(async () => {
|
||||
env = await setupPuterTestEnv();
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await env?.shutdown();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
// Hold the same cached snapshot until every concurrent request has read it.
|
||||
const overlapReads = (userUuid: string, count: number) => {
|
||||
const redis = env.server.clients.redis;
|
||||
const get = redis.get.bind(redis);
|
||||
const ready = Promise.withResolvers<void>();
|
||||
let reads = 0;
|
||||
vi.spyOn(redis, 'get').mockImplementation(async (key: string) => {
|
||||
const value = await get(key);
|
||||
if (key.includes(userUuid) && reads < count) {
|
||||
if (++reads === count) ready.resolve();
|
||||
await ready.promise;
|
||||
}
|
||||
return value;
|
||||
});
|
||||
};
|
||||
|
||||
const post = async (path: string, body: Record<string, unknown>) => {
|
||||
const response = await fetch(new URL(path, env.origin), {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return { response, body: await response.json() };
|
||||
};
|
||||
|
||||
const account = async (codes: string[]) => {
|
||||
const username = `recovery_${uuidv4().slice(0, 8)}`;
|
||||
await createTestUser(env.server, { username, password });
|
||||
const user = (await env.server.stores.user.getByUsername(username))!;
|
||||
await env.server.stores.user.update(user.id, {
|
||||
otp_enabled: true,
|
||||
otp_secret: 'TESTSECRETBASE32',
|
||||
otp_recovery_codes: codes.map(hashRecoveryCode).join(','),
|
||||
});
|
||||
const challenge = async () => {
|
||||
const { response, body } = await post('/login', {
|
||||
username,
|
||||
password,
|
||||
});
|
||||
expect(response.status).toBe(202);
|
||||
expect(body.next_step).toBe('otp');
|
||||
return body.otp_jwt_token as string;
|
||||
};
|
||||
return { user, challenge };
|
||||
};
|
||||
|
||||
it('allows only one concurrent redemption of the same code', async () => {
|
||||
const { user, challenge } = await account(['RECOVER1', 'RECOVER2']);
|
||||
const tokens = await Promise.all([challenge(), challenge()]);
|
||||
overlapReads(user.uuid, tokens.length);
|
||||
const results = await Promise.all(
|
||||
tokens.map((token) =>
|
||||
post('/login/recovery-code', { token, code: 'RECOVER1' }),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
results.filter((result) => result.body.next_step === 'complete'),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
results.filter((result) => result.body.proceed === false),
|
||||
).toHaveLength(1);
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.otp_recovery_codes).toBe(hashRecoveryCode('RECOVER2'));
|
||||
});
|
||||
|
||||
it('consumes different concurrent codes without restoring either one', async () => {
|
||||
const codes = ['RECOVER1', 'RECOVER2', 'RECOVER3'];
|
||||
const { user, challenge } = await account(codes);
|
||||
const tokens = await Promise.all(codes.map(() => challenge()));
|
||||
overlapReads(user.uuid, tokens.length);
|
||||
const results = await Promise.all(
|
||||
codes.map((code, index) =>
|
||||
post('/login/recovery-code', { token: tokens[index], code }),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
results.every((result) => result.body.next_step === 'complete'),
|
||||
).toBe(true);
|
||||
const after = await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
});
|
||||
expect(after!.otp_recovery_codes).toBe('');
|
||||
const replay = await post('/login/recovery-code', {
|
||||
token: tokens[0],
|
||||
code: codes[0],
|
||||
});
|
||||
expect(replay.body).toMatchObject({ proceed: false });
|
||||
});
|
||||
|
||||
it.each([{ code: 123456 }, { code: {} }, { code: [] }])(
|
||||
'rejects a non-string recovery code $code without consuming a code',
|
||||
async ({ code }) => {
|
||||
const { user, challenge } = await account(['RECOVER1']);
|
||||
const result = await post('/login/recovery-code', {
|
||||
token: await challenge(),
|
||||
code,
|
||||
});
|
||||
expect(result.response.status).toBe(400);
|
||||
expect(
|
||||
(await env.server.stores.user.getById(user.id, {
|
||||
force: true,
|
||||
}))!.otp_recovery_codes,
|
||||
).toBe(hashRecoveryCode('RECOVER1'));
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,82 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { Request, Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { configContainer } from '../../exports.js';
|
||||
import { PuterServer } from '../../server.js';
|
||||
import { setupTestServer } from '../../testUtil.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
import { startWebSession } from './webSession.js';
|
||||
|
||||
let server: PuterServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
describe('startWebSession', () => {
|
||||
it('records the request on the session, sets the HTTP-only cookie and returns a GUI token', async () => {
|
||||
const username = `ws_${Math.random().toString(36).slice(2, 10)}`;
|
||||
const user = await server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
});
|
||||
const cookies: Array<{ name: string; value: string; opts: unknown }> =
|
||||
[];
|
||||
const res = {
|
||||
cookie: (name: string, value: string, opts: unknown) => {
|
||||
cookies.push({ name, value, opts });
|
||||
},
|
||||
} as unknown as Response;
|
||||
const req = {
|
||||
ip: '192.0.2.10',
|
||||
headers: { 'user-agent': 'ws-agent', origin: 'https://gui.test' },
|
||||
} as unknown as Request;
|
||||
const config = configContainer as IConfig;
|
||||
|
||||
const guiToken = await startWebSession(req, res, user, {
|
||||
config,
|
||||
auth: server.services.auth,
|
||||
});
|
||||
|
||||
expect(typeof guiToken).toBe('string');
|
||||
expect(cookies).toHaveLength(1);
|
||||
expect(cookies[0]!.name).toBe(config.cookie_name ?? 'puter_token');
|
||||
expect(cookies[0]!.value).not.toBe(guiToken);
|
||||
expect(cookies[0]!.opts).toMatchObject({ httpOnly: true });
|
||||
|
||||
const sessions = (await server.stores.session.getByUserId(
|
||||
user.id,
|
||||
)) as Array<Record<string, unknown>>;
|
||||
expect(sessions).toHaveLength(1);
|
||||
expect(sessions[0]).toMatchObject({
|
||||
kind: 'web',
|
||||
last_ip: '192.0.2.10',
|
||||
last_user_agent: 'ws-agent',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,48 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { Request, Response } from 'express';
|
||||
import type { AuthService } from '../../services/auth/AuthService.js';
|
||||
import type { UserRow } from '../../stores/user/UserStore.js';
|
||||
import type { IConfig } from '../../types.js';
|
||||
import { sessionCookieFlags } from '../../util/cookieFlags.js';
|
||||
|
||||
/**
|
||||
* Open a web session for a sign-in this request completed, recording where it
|
||||
* came from, and set its HTTP-only cookie. Returns the GUI token, the only
|
||||
* credential the client sees.
|
||||
*/
|
||||
export async function startWebSession(
|
||||
req: Request,
|
||||
res: Response,
|
||||
user: UserRow,
|
||||
deps: { config: IConfig; auth: AuthService },
|
||||
): Promise<string> {
|
||||
const { token, gui_token } = await deps.auth.createSessionToken(user, {
|
||||
ip: req.ip || req.socket?.remoteAddress,
|
||||
user_agent: req.headers?.['user-agent'],
|
||||
origin: req.headers?.origin,
|
||||
host: req.headers?.host,
|
||||
});
|
||||
res.cookie(deps.config.cookie_name ?? 'puter_token', token, {
|
||||
...sessionCookieFlags(deps.config),
|
||||
httpOnly: true,
|
||||
});
|
||||
return gui_token;
|
||||
}
|
||||
@@ -73,8 +73,8 @@ export class BroadcastController extends PuterController {
|
||||
/**
|
||||
* Event deliveries one region addressed at this one, on the same signed
|
||||
* channel as the webhook above but carrying socket traffic rather than bus
|
||||
* events. The answer names the anchor tokens this region holds no session
|
||||
* row for, which is what lets the sender prune its remote-watch index.
|
||||
* events. The answer names the pairs this region holds no socket for, which
|
||||
* is what lets the sender correct its presence row.
|
||||
*/
|
||||
@Post('/events', { subdomain: '*' })
|
||||
async events(req: Request, res: Response): Promise<void> {
|
||||
|
||||
@@ -46,6 +46,17 @@ const PREFERENCE_WRITE_LIMIT = {
|
||||
};
|
||||
|
||||
export class DesktopController extends PuterController {
|
||||
constructor(config, clients, stores, services) {
|
||||
super(config, clients, stores, services);
|
||||
}
|
||||
|
||||
get userStore() {
|
||||
return this.stores.user;
|
||||
}
|
||||
get db() {
|
||||
return this.clients.db;
|
||||
}
|
||||
|
||||
registerRoutes(router) {
|
||||
// -- Desktop background --------------------------------------
|
||||
|
||||
@@ -98,7 +109,7 @@ export class DesktopController extends PuterController {
|
||||
});
|
||||
}
|
||||
|
||||
await this.stores.user.update(req.actor.user.id, patch);
|
||||
await this.userStore.update(req.actor.user.id, patch);
|
||||
res.json({});
|
||||
},
|
||||
);
|
||||
@@ -123,7 +134,7 @@ export class DesktopController extends PuterController {
|
||||
);
|
||||
}
|
||||
|
||||
await this.stores.user.update(req.actor.user.id, {
|
||||
await this.userStore.update(req.actor.user.id, {
|
||||
taskbar_items: JSON.stringify(items),
|
||||
});
|
||||
res.json({});
|
||||
@@ -239,7 +250,7 @@ export class DesktopController extends PuterController {
|
||||
const setClause = keys.map((k) => `\`${k}\` = ?`).join(', ');
|
||||
const values = keys.map((k) => patch[k]);
|
||||
|
||||
await this.clients.db.write(
|
||||
await this.db.write(
|
||||
`UPDATE \`fsentries\` SET ${setClause} WHERE \`id\` = ?`,
|
||||
[...values, entry.id],
|
||||
);
|
||||
|
||||
@@ -262,22 +262,6 @@ describe('DriverController — concurrent acquire/release', () => {
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
it('releases a slot acquired after the client has disconnected', async () => {
|
||||
const handler = captureCallHandler(buildController(makeSyntheticDriver()));
|
||||
const res1 = new StubRes();
|
||||
Object.assign(res1, { destroyed: true });
|
||||
await callInScope(handler,
|
||||
makeReq({ interface: 'test-iface', method: 'ping' }, 'already-aborted'),
|
||||
res1 as unknown as Response);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const res2 = new StubRes();
|
||||
await callInScope(handler,
|
||||
makeReq({ interface: 'test-iface', method: 'ping' }, 'already-aborted'),
|
||||
res2 as unknown as Response);
|
||||
res2.emit('finish');
|
||||
expect(res2.body).toMatchObject({ success: true, result: 'pong' });
|
||||
});
|
||||
|
||||
it('does not attach release listeners when the driver declares no concurrent config', async () => {
|
||||
// With no spec there is nothing to release, so the gate must not
|
||||
// wire `finish`/`close` release listeners. The client-disconnect
|
||||
|
||||
Loaded 100 of 1376 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user