Compare commits

...
Author SHA1 Message Date
ProgrammerIn-wonderland cca509f821 PUT-1903 2026-10-13 17:30:29 -04:00
Neal ShahandClaude Opus 5.5 4401b4c4dc Fix worker test (#4156)
* PUT-2012 fix broadcast service drops

* Revert pre-commit lint churn on files merged from main

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

* fix test bug

* Remove other test

---------

Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-13 14:10:10 -04:00
Neal Shah 62270ff9aa Delete worker sessions when workers are deleted, don't treat sessions from deleted workers as still valid. (#4163) 2026-10-13 12:43:45 -04:00
27 changed files with 1339 additions and 99 deletions

No files matched your search

@@ -2259,6 +2259,62 @@ describe('AuthController grant flows', () => {
).not.toContain(permission);
});
it('revoke-user-app `*`: deletes the worker sessions that app deployed for the user, and only those', async () => {
const app = await server.stores.app.create(
{
name: `tr-worker-${uuidv4()}`,
title: 'TestRevokeWorkerApp',
index_url: `https://tr-worker-${uuidv4()}.example.test/`,
},
{ ownerUserId: issuer.id },
);
const workerName = () => `wk-${uuidv4().slice(0, 8)}`;
const appWorker = await server.stores.session.getOrCreateWorker(
issuer.id,
{ appUid: app.uid, workerName: workerName() },
);
const otherApp = await server.stores.session.getOrCreateWorker(
issuer.id,
{ appUid: `app-${uuidv4()}`, workerName: workerName() },
);
const userScoped = await server.stores.session.getOrCreateWorker(
issuer.id,
{ appUid: null, workerName: workerName() },
);
const exists = async (uuid: string) =>
(
await server.clients.db.read(
'SELECT 1 FROM `sessions` WHERE `uuid` = ?',
[uuid],
)
).length > 0;
// A single permission leaves the app's workers running as before.
await inCtx(issuerActor, () =>
controller.handleRevokeUserApp(
makeReq(
{ app_uid: app.uid, permission: 'service:x:ii:read' },
{ actor: issuerActor },
),
makeRes(),
),
);
expect(await exists(appWorker.uuid)).toBe(true);
await inCtx(issuerActor, () =>
controller.handleRevokeUserApp(
makeReq(
{ app_uid: app.uid, permission: '*' },
{ actor: issuerActor },
),
makeRes(),
),
);
expect(await exists(appWorker.uuid)).toBe(false);
expect(await exists(otherApp.uuid)).toBe(true);
expect(await exists(userScoped.uuid)).toBe(true);
});
it('grant-user-app: 400 on non-string or oversized origin/app_uid/permission', async () => {
const cases = [
{ origin: { host: 'evil' }, permission: 'service:x:ii:read' },
@@ -3265,6 +3265,12 @@ export class AuthController extends PuterController {
app_uid,
meta ?? undefined,
);
// Withdrawing everything also retires the workers the app deployed
// under this user; their tokens would otherwise act as the app.
await this.services.auth.deleteWorkerSessionsForApp(
req.actor!.user!.id as number,
app_uid,
);
} else {
for (const entry of list) {
await this.services.permission.revokeUserAppPermission(
@@ -115,6 +115,31 @@ async function checkMemory(key, limit, windowMs) {
return true;
}
/**
* Spend up to `units` from one bucket at once and report how many fit: the
* many-unit form of `checkMemory`, for a caller that would otherwise loop.
*/
async function claimMemory(key, limit, windowMs, units) {
const now = Date.now();
let entry = memoryWindows.get(key);
if (!entry) {
if (memoryWindows.size >= MEMORY_MAX_KEYS) {
const oldest = memoryWindows.keys().next().value;
memoryWindows.delete(oldest);
}
entry = { ts: [], windowMs };
memoryWindows.set(key, entry);
} else {
entry.windowMs = windowMs;
}
const timestamps = entry.ts;
const cutoff = now - windowMs;
while (timestamps.length > 0 && timestamps[0] < cutoff) timestamps.shift();
const admitted = Math.max(0, Math.min(units, limit - timestamps.length));
for (let i = 0; i < admitted; i++) timestamps.push(now);
return admitted;
}
/**
* Read a bucket's state without spending from it. For gates whose budget is
* consumed by something other than the request being admitted — a failed
@@ -182,6 +207,32 @@ async function checkRedis(
return true;
}
/**
* `checkRedis` for many units in one round trip: add them all, count, and take
* back however many landed over the limit. Same add-then-count shape, so
* concurrent claims can under-admit but never over-admit.
*/
async function claimRedis(redis, key, limit, windowMs, units) {
const redisKey = `rate:${key}`;
const now = Date.now();
const members = Array.from(
{ length: units },
() => `${now}:${crypto.randomUUID()}`,
);
const results = await redis
.multi()
.zremrangebyscore(redisKey, 0, now - windowMs)
.zadd(redisKey, ...members.flatMap((member) => [now, member]))
.zcard(redisKey)
.pexpire(redisKey, windowMs)
.exec();
const count = Number(multiResult(results, 2));
const excess = Math.min(units, Math.max(0, count - limit));
if (excess > 0) await redis.zrem(redisKey, ...members.slice(-excess));
return units - excess;
}
async function peekRedis(redis, key, limit, windowMs) {
const redisKey = `rate:${key}`;
const results = await redis
@@ -218,6 +269,31 @@ async function checkKv(kv, key, limit, windowMs) {
return true;
}
// `checkKv` for many units: one read, then a row per admitted unit. Newest
// first for the same reason `checkKv` reads that way.
async function claimKv(kv, key, limit, windowMs, units) {
const prefix = `rate:${key}:`;
const { res } = await kv.list({
as: 'keys',
pattern: prefix,
limit: limit + 1,
reverse: true,
});
const keys = Array.isArray(res) ? res : (res?.items ?? []);
const admitted = Math.max(0, Math.min(units, limit - keys.length));
const now = Date.now();
await Promise.all(
Array.from({ length: admitted }, () =>
kv.set({
key: `${prefix}${now}:${crypto.randomUUID()}`,
value: 1,
expireAt: Math.ceil((now + windowMs) / 1000),
}),
),
);
return admitted;
}
// `list` filters by TTL, so a non-expired row is in-window — same read
// `checkKv` does, minus the write.
async function peekKv(kv, key, limit) {
@@ -395,6 +471,10 @@ function instrumentBackendPair(name, pair) {
withSpan('rate_limit.peek', attrs, () =>
pair.peek(key, limit, windowMs),
),
claim: (key, limit, windowMs, units) =>
withSpan('rate_limit.claim', attrs, () =>
pair.claim(key, limit, windowMs, units),
),
acquire: (key, limit) =>
withSpan('rate_limit.acquire', attrs, () =>
pair.acquire(key, limit),
@@ -405,6 +485,7 @@ function instrumentBackendPair(name, pair) {
const memoryBackendPair = instrumentBackendPair('memory', {
rate: checkMemory,
peek: peekMemory,
claim: claimMemory,
acquire: acquireMemoryConcurrent,
});
@@ -453,6 +534,8 @@ export function configureRateLimit({
checkRedis(redis, key, limit, windowMs),
peek: (key, limit, windowMs) =>
peekRedis(redis, key, limit, windowMs),
claim: (key, limit, windowMs, units) =>
claimRedis(redis, key, limit, windowMs, units),
acquire: (key, limit) => acquireRedisConcurrent(redis, key, limit),
});
}
@@ -460,6 +543,8 @@ export function configureRateLimit({
backends.kv = instrumentBackendPair('kv', {
rate: (key, limit, windowMs) => checkKv(kv, key, limit, windowMs),
peek: (key, limit) => peekKv(kv, key, limit),
claim: (key, limit, windowMs, units) =>
claimKv(kv, key, limit, windowMs, units),
acquire: (key, limit) => acquireKvConcurrent(kv, key, limit),
});
}
@@ -770,6 +855,25 @@ export async function checkRateLimit(
);
}
/**
* `checkRateLimit` for `units` hits at once, in one backend round trip.
* Resolves to how many were admitted (0..units) and spent; the rest were not
* charged. Fails open, admitting every unit, like the rest of this module.
*/
export async function claimRateLimit(key, limit, windowMs, units, backend) {
if (!(units > 0)) return 0;
const bk = resolveBackend(backend);
try {
return await bk.claim(key, limit, windowMs, units);
} catch (err) {
console.error(
'[rate-limit] imperative claim failed, failing open:',
err,
);
return units;
}
}
/**
* Charge a route-shaped rate-limit spec from inside a handler.
*
@@ -37,6 +37,7 @@ import {
acquireDriverConcurrent,
checkDriverRateLimit,
checkRateLimit,
claimRateLimit,
concurrencyGate,
consumeRouteRateLimit,
configureRateLimit,
@@ -1479,6 +1480,20 @@ describe('rate + concurrent limiting — kv backend', () => {
await Promise.all(held.map((slot) => slot.release()));
});
it('claims many units in one call, admitting only what fits', async () => {
const key = uniqueScope();
expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(3);
expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(2);
expect(await claimRateLimit(key, 5, 60_000, 1)).toBe(0);
});
it('claims against a full bucket as full while expired rows are unswept', async () => {
const key = uniqueScope();
await seedUnswept(`rate:${key}:`, 3);
expect(await claimRateLimit(key, 3, 60_000, 3)).toBe(3);
expect(await claimRateLimit(key, 3, 60_000, 1)).toBe(0);
});
it('holds a concurrent slot until it is released', async () => {
const req = { ip: '9.9.9.9', headers: {}, actor: undefined };
const opts = { limit: 1, backend: 'kv', scope: uniqueScope() };
@@ -1994,3 +2009,59 @@ describe('acquireConcurrent (imperative)', () => {
configureRateLimit();
});
});
describe('claimRateLimit', () => {
const uniqueKey = () => `claim-${Math.random().toString(36).slice(2, 10)}`;
describe.each([
['memory', () => configureRateLimit()],
[
'redis',
() => configureRateLimit({ default: 'redis', redis: new RedisMock() }),
],
])('%s backend', (_name, configure) => {
beforeEach(configure);
it('admits every unit while under the limit', async () => {
expect(await claimRateLimit(uniqueKey(), 5, 60_000, 3)).toBe(3);
});
it('admits exactly up to the limit and spends only that', async () => {
const key = uniqueKey();
expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(3);
expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(2);
// Nothing spent past the limit, so the bucket is exactly full.
expect(await checkRateLimit(key, 5, 60_000)).toBe(false);
expect(await checkRateLimit(key, 6, 60_000)).toBe(true);
});
it('admits nothing once the bucket is full', async () => {
const key = uniqueKey();
expect(await claimRateLimit(key, 2, 60_000, 2)).toBe(2);
expect(await claimRateLimit(key, 2, 60_000, 4)).toBe(0);
});
it('shares its bucket with checkRateLimit', async () => {
const key = uniqueKey();
expect(await checkRateLimit(key, 3, 60_000)).toBe(true);
expect(await claimRateLimit(key, 3, 60_000, 5)).toBe(2);
});
});
it('claims nothing for zero units', async () => {
const key = uniqueKey();
expect(await claimRateLimit(key, 1, 60_000, 0)).toBe(0);
expect(await checkRateLimit(key, 1, 60_000)).toBe(true);
});
it('fails open, admitting every unit, when the backend throws', async () => {
const redis = new RedisMock();
redis.multi = () => {
throw new Error('redis down');
};
configureRateLimit({ default: 'redis', redis });
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
expect(await claimRateLimit(uniqueKey(), 1, 60_000, 4)).toBe(4);
spy.mockRestore();
});
});
@@ -926,3 +926,108 @@ describe('WorkerDriver hot reload', () => {
expect(sessionMint).not.toHaveBeenCalled();
});
});
// -- worker sessions --------------------------------------------------
describe('WorkerDriver keeps a worker`s token tied to the worker', () => {
const deployedToken = () => {
const [, init] = putCalls().at(-1)!;
const form = (init as RequestInit).body as FormData;
const metadata = JSON.parse(form.get('metadata') as string);
return metadata.bindings.find(
(b: { name: string }) => b.name === 'puter_auth',
).text as string;
};
const deployWorker = async (label: string) => {
const { user, actor } = await makeUser();
const path = `/${user.username}/${label}.js`;
const entry = await writeSource(actor, user.id, path, 'src');
const name = `${label}-${user.username}`;
await inCtx(actor, () =>
target.create({ appId: '', workerName: name, filePath: path }),
);
return { user, actor, entry, name, token: deployedToken() };
};
const workerSession = (userId: number, workerName: string) =>
server.stores.session.getWorker(userId, { appUid: null, workerName });
it('binds the deployed token to the worker row, and a redeploy keeps it', async () => {
const { user, actor, name, token } = await deployWorker('bind');
const row = await server.stores.subdomain.getBySubdomain(
`workers.puter.${name}`,
);
const decoded = server.services.token.verify('auth', token) as {
worker_uid?: string;
};
expect(decoded.worker_uid).toBe(row!.uuid);
expect(
(await server.services.auth.authenticate(token)).actor,
).toBeTruthy();
await inCtx(actor, () =>
target.create({
appId: '',
workerName: name,
filePath: `/${user.username}/bind.js`,
}),
);
const redeployed = server.services.token.verify(
'auth',
deployedToken(),
) as { worker_uid?: string };
expect(redeployed.worker_uid).toBe(row!.uuid);
});
it('destroy deletes the worker`s session and leaves other workers alone', async () => {
const kept = await deployWorker('kept');
const { user, actor, name, token } = await deployWorker('gone');
await inCtx(actor, () => target.destroy({ workerName: name }));
expect(await workerSession(user.id, name)).toBeNull();
expect(await server.services.auth.authenticate(token)).toEqual({
invalid: true,
});
expect(await workerSession(kept.user.id, kept.name)).toBeTruthy();
expect(
(await server.services.auth.authenticate(kept.token)).actor,
).toBeTruthy();
});
it('deleting the source file deletes the worker`s session', async () => {
const { user, actor, entry, name } = await deployWorker('srcdel');
// Resolved at delete time, as every caller does: the entry names the
// workers bound to it.
const current = await server.stores.fsEntry.getEntryByUuid(entry.uuid);
await inCtx(actor, () =>
server.services.fs.remove(user.id, { entry: current! }),
);
await waitFor(
async () => !(await workerSession(user.id, name)),
'worker session removal after source delete',
);
});
it('a worker row deleted elsewhere takes its session with it', async () => {
const { user, name } = await deployWorker('elsewhere');
const row = await server.stores.subdomain.getBySubdomain(
`workers.puter.${name}`,
);
await server.stores.subdomain.deleteByUuid(String(row!.uuid));
server.clients.event.emit(
'subdomain.delete',
{ subdomain: `workers.puter.${name}`, uid: String(row!.uuid) },
{},
);
await waitFor(
async () => !(await workerSession(user.id, name)),
'worker session removal after subdomain.delete',
);
});
});
@@ -20,6 +20,7 @@
import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { v4 as uuidv4 } from 'uuid';
import type { EventMetadata } from '../../clients/event/types.js';
import {
assertActorEmailVerified,
@@ -414,11 +415,19 @@ export class WorkerDriver extends PuterDriver {
let authorization: string | undefined = undefined;
const appOwnerId = boundApp?.id;
const omitOwnerToken = deployTarget?.omitOwnerToken === true;
// The token is bound to the row it deploys under, which a new worker
// only gets further down, so its uuid is picked here.
const workerUid = deployTarget
? undefined
: existingSub
? String(existingSub.uuid)
: uuidv4();
if (boundApp && !omitOwnerToken) {
authorization = await this.services.auth.createWorkerAppToken(
actor,
boundApp.uid,
workerName,
{ workerUid },
);
}
if (!authorization && !omitOwnerToken) {
@@ -434,6 +443,8 @@ export class WorkerDriver extends PuterDriver {
actor,
userRow,
workerName,
{},
{ workerUid },
);
authorization = session.token;
}
@@ -490,6 +501,7 @@ export class WorkerDriver extends PuterDriver {
// than letting the driver error escape as a 500.
try {
await this.stores.subdomain.create({
uuid: workerUid,
userId: actor.user.id!,
subdomain: subdomainName,
rootDirId: loaded.fsEntry.sqlId,
@@ -572,6 +584,7 @@ export class WorkerDriver extends PuterDriver {
await this.stores.subdomain.deleteByUuid(row.uuid, {
userId: actor.user.id,
});
await this.services.auth.deleteWorkerSessionsByName(workerName);
return cfResult;
}
@@ -1076,6 +1089,14 @@ export class WorkerDriver extends PuterDriver {
err,
);
});
void this.services.auth
.deleteWorkerSessionsByName(workerName)
.catch((err) => {
console.warn(
`[workers] session cleanup failed for ${workerName}`,
err,
);
});
},
);
}
@@ -1145,6 +1166,11 @@ export class WorkerDriver extends PuterDriver {
ownerActor,
app.uid,
workerName,
{
workerUid: row.uuid
? String(row.uuid)
: undefined,
},
);
} else {
const session =
@@ -1152,6 +1178,12 @@ export class WorkerDriver extends PuterDriver {
ownerActor,
ownerUser,
workerName,
{},
{
workerUid: row.uuid
? String(row.uuid)
: undefined,
},
);
authorization = session.token;
@@ -1325,6 +1357,7 @@ export class WorkerDriver extends PuterDriver {
userId,
});
}
await this.services.auth.deleteWorkerSessionsByName(workerName);
} catch (err) {
console.warn(
`[workers] source cleanup failed for ${workerName}`,
@@ -26,6 +26,7 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { v4 as uuidv4 } from 'uuid';
import { makeActor } from '../../core/actor.js';
import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
import {
createTestUser,
setupPuterTestEnv,
@@ -327,10 +328,16 @@ describe('an app uid that returns after its app was deleted', () => {
it('replaces a worker session older than its app', async () => {
const { app, actor } = await createApp();
const workerName = `w-${uuidv4().slice(0, 8)}`;
const { uuid: workerUid } = await env.server.stores.subdomain.create({
userId: actor.user.id as number,
subdomain: `${WORKER_SUBDOMAIN_PREFIX}${workerName}`,
appOwner: app.id,
});
const old = await env.server.services.auth.createWorkerAppToken(
actor,
app.uid,
workerName,
{ workerUid },
);
await backdateSession(sessionUid(old), 3600);
expect((await authenticate(old)).reauth?.reason).toBe(
@@ -341,6 +348,7 @@ describe('an app uid that returns after its app was deleted', () => {
actor,
app.uid,
workerName,
{ workerUid },
);
expect(sessionUid(fresh)).not.toBe(sessionUid(old));
expect(await isRevoked(sessionUid(old))).toBe(true);
@@ -22,8 +22,10 @@ import { v4 as uuidv4, v5 as uuidv5 } from 'uuid';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { makeActor, type Actor } from '../../core/actor.js';
import { PuterServer } from '../../server.js';
import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
import { setupTestServer } from '../../testUtil.js';
import { generateDefaultFsentries } from '../../util/userProvisioning.js';
import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
import { FULL_API_ACCESS } from '../permission/consts.js';
import { AuthService, HOSTED_ASSET_COOKIES } from './AuthService.js';
@@ -1647,10 +1649,16 @@ describe('AuthService (integration)', () => {
// revoke so the re-create produces a fresh row.
const user = await makeUser();
const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
const { uuid: workerUid } = await server.stores.subdomain.create({
userId: user.id,
subdomain: `${WORKER_SUBDOMAIN_PREFIX}${workerName}`,
});
const first = await authService.createWorkerSessionToken(
makeActor({ user }),
user,
workerName,
{},
{ workerUid },
);
const firstUuid = (first.session as { uuid: string }).uuid;
await authService.revokeSession(firstUuid);
@@ -1659,6 +1667,8 @@ describe('AuthService (integration)', () => {
makeActor({ user }),
user,
workerName,
{},
{ workerUid },
);
const secondUuid = (second.session as { uuid: string }).uuid;
expect(secondUuid).not.toBe(firstUuid);
@@ -1731,6 +1741,178 @@ describe('AuthService (integration)', () => {
});
});
describe('a worker token lives only as long as its worker', () => {
const deployRow = async (
userId: number,
workerName: string,
appOwner?: number,
) =>
await server.stores.subdomain.create({
userId,
subdomain: `${WORKER_SUBDOMAIN_PREFIX}${workerName}`,
appOwner: appOwner ?? null,
});
const newName = () => `wk-${Math.random().toString(36).slice(2, 8)}`;
it('resolves while the worker row exists and is invalid once it is deleted', async () => {
const user = await makeUser();
const workerName = newName();
const row = await deployRow(user.id, workerName);
const { token, gui_token } =
await authService.createWorkerSessionToken(
makeActor({ user }),
user,
workerName,
{},
{ workerUid: row.uuid },
);
expect((await authService.authenticate(token)).actor).toBeTruthy();
// Straight through the store: no hook revokes the session.
await server.stores.subdomain.deleteByUuid(row.uuid);
expect(await authService.authenticate(token)).toEqual({
invalid: true,
});
expect(await authService.authenticate(gui_token)).toEqual({
invalid: true,
});
});
it('does not come back when the same name is deployed again', async () => {
const user = await makeUser();
const workerName = newName();
const first = await deployRow(user.id, workerName);
const { token } = await authService.createWorkerSessionToken(
makeActor({ user }),
user,
workerName,
{},
{ workerUid: first.uuid },
);
await server.stores.subdomain.deleteByUuid(first.uuid);
await deployRow(user.id, workerName);
expect(await authService.authenticate(token)).toEqual({
invalid: true,
});
});
it('applies to app-scoped worker tokens', async () => {
const user = await makeUser();
const app = await server.stores.app.create(
{
name: `wk-app-${uuidv4()}`,
title: 'Worker app',
index_url: `https://${uuidv4()}.example.com/`,
},
{ ownerUserId: user.id },
);
const workerName = newName();
const row = await deployRow(user.id, workerName, app.id);
const token = await authService.createWorkerAppToken(
makeActor({ user }),
app.uid,
workerName,
{ workerUid: row.uuid },
);
expect(
(await authService.authenticate(token)).actor?.app?.uid,
).toBe(app.uid);
await server.stores.subdomain.deleteByUuid(row.uuid);
expect(await authService.authenticate(token)).toEqual({
invalid: true,
});
});
it('holds a token minted without a worker uid to the name and owner', async () => {
const user = await makeUser();
const other = await makeUser();
const workerName = newName();
const { token } = await authService.createWorkerSessionToken(
makeActor({ user }),
user,
workerName,
);
// No row at all.
expect(await authService.authenticate(token)).toEqual({
invalid: true,
});
// Someone else's worker under that name.
const foreign = await deployRow(other.id, workerName);
expect(await authService.authenticate(token)).toEqual({
invalid: true,
});
await server.stores.subdomain.deleteByUuid(foreign.uuid);
await deployRow(user.id, workerName);
expect((await authService.authenticate(token)).actor).toBeTruthy();
});
it('leaves events handler sessions alone; they have no worker row', async () => {
const user = await makeUser();
const app = await server.stores.app.create(
{
name: `ev-app-${uuidv4()}`,
title: 'Events app',
index_url: `https://${uuidv4()}.example.com/`,
},
{ ownerUserId: user.id },
);
const token = await authService.createWorkerAppToken(
makeActor({ user }),
app.uid,
EVENTS_WORKER_SESSION_NAME,
);
expect((await authService.authenticate(token)).actor).toBeTruthy();
});
});
describe('deleting worker sessions', () => {
it('deleteWorkerSessionsByName ends the token and drops a derived token`s grants', async () => {
const user = await makeUser();
const workerName = `wk-${Math.random().toString(36).slice(2, 8)}`;
const row = await server.stores.subdomain.create({
userId: user.id,
subdomain: `${WORKER_SUBDOMAIN_PREFIX}${workerName}`,
});
const { session, token } =
await authService.createWorkerSessionToken(
makeActor({ user }),
user,
workerName,
{},
{ workerUid: row.uuid },
);
const sessionUuid = (session as { uuid: string }).uuid;
const childUid = uuidv4();
await server.stores.session.create(user.id, {
kind: 'access_token',
parent_session_id: sessionUuid,
access_token_uid: childUid,
});
await server.clients.db.write(
'INSERT INTO `access_token_permissions` (`token_uid`, `authorizer_user_id`, `permission`) VALUES (?, ?, ?)',
[childUid, user.id, 'fs:x:read'],
);
await authService.deleteWorkerSessionsByName(workerName);
// The worker row is still there: only the session went.
expect(await authService.authenticate(token)).toEqual({
invalid: true,
});
const grants = await server.clients.db.read(
'SELECT * FROM `access_token_permissions` WHERE `token_uid` = ?',
[childUid],
);
expect(grants).toHaveLength(0);
});
});
describe('appUidFromOrigin', () => {
it('throws 400 for an unparseable origin string', async () => {
await expect(
+95 -6
View File
@@ -33,12 +33,14 @@ import {
GODMODE_TOKEN_WINDOW_SECONDS,
WEB_WINDOW_SECONDS,
} from '../../stores/session/SessionStore.js';
import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js';
import type { UserRow } from '../../stores/user/UserStore';
import type { LayerInstances } from '../../types';
import { sessionCookieFlags } from '../../util/cookieFlags.js';
import { isGodmodeApp } from '../../util/godmodeApps.js';
import { Span } from '../../util/span.js';
import type { puterServices } from '../index';
import { EVENTS_WORKER_SESSION_NAME } from '../events/workerRuntime.js';
import { FULL_API_ACCESS, PERMISSION_MAX_LEN } from '../permission/consts';
import { PuterService } from '../types';
import type {
@@ -401,7 +403,11 @@ export class AuthService extends PuterService {
user: UserRow,
sessionUuid: string,
authId: string,
opts: { worker?: boolean; workerName?: string } = {},
opts: {
worker?: boolean;
workerName?: string;
workerUid?: string;
} = {},
): string {
const claims: Record<string, unknown> = {
type,
@@ -416,6 +422,7 @@ export class AuthService extends PuterService {
};
if (opts.worker) claims.worker = true;
if (opts.workerName) claims.worker_name = opts.workerName;
if (opts.workerUid) claims.worker_uid = opts.workerUid;
return this.services.token.sign('auth', claims);
}
@@ -426,13 +433,15 @@ export class AuthService extends PuterService {
* the row and returns the same stable token. Expires after
* `WORKER_WINDOW_SECONDS` (effectively infinite). The emitted JWT carries
* `worker: true` and `worker_name` so downstream code can tell a worker
* session from a user-driven one without a DB round-trip.
* session from a user-driven one without a DB round-trip. `workerUid` binds
* the token to the worker's subdomain row, so it stops working with it.
*/
async createWorkerSessionToken(
actor: Actor,
user: UserRow,
workerName: string,
meta: Record<string, unknown> = {},
{ workerUid }: { workerUid?: string } = {},
): Promise<{
session: Record<string, unknown>;
token: string;
@@ -464,14 +473,14 @@ export class AuthService extends PuterService {
user,
session.uuid as string,
auth_id,
{ worker: true, workerName },
{ worker: true, workerName, workerUid },
);
const gui_token = this.#signSessionTypeToken(
'gui',
user,
session.uuid as string,
auth_id,
{ worker: true, workerName },
{ worker: true, workerName, workerUid },
);
return { session, token, gui_token };
@@ -489,13 +498,18 @@ export class AuthService extends PuterService {
* handler runs deep; `expiresInSeconds` bounds a token that carries one.
* Refuses an actor that is itself running behind a handler — the minted
* token would outlive any one run, so stamping it with the caller's depth
* would only delay the escape, not close it.
* would only delay the escape, not close it. `workerUid` binds the token to
* the worker's subdomain row, so it stops working with it.
*/
async createWorkerAppToken(
actor: Actor,
appUid: string,
workerName: string,
options: { handlerDepth?: number; expiresInSeconds?: number } = {},
options: {
handlerDepth?: number;
expiresInSeconds?: number;
workerUid?: string;
} = {},
): Promise<string> {
if (!actor.user) {
throw new HttpError(403, 'Actor must be a user', {
@@ -546,6 +560,7 @@ export class AuthService extends PuterService {
auth_id,
worker: true,
worker_name: workerName,
...(options.workerUid ? { worker_uid: options.workerUid } : {}),
...(options.handlerDepth
? { handler_depth: options.handlerDepth }
: {}),
@@ -891,6 +906,50 @@ export class AuthService extends PuterService {
}
}
/**
* Delete every session of a worker that no longer exists, whoever minted
* it: worker names are unique, so nothing else can be using them.
*/
async deleteWorkerSessionsByName(workerName: string): Promise<void> {
if (!workerName) return;
await this.#deleteWorkerSessions({ workerName });
}
/**
* Delete the sessions of the workers an app deployed for this user, e.g.
* when the user uninstalls it. The workers themselves keep running.
*/
async deleteWorkerSessionsForApp(
userId: number,
appUid: string,
): Promise<void> {
if (!userId || !appUid) return;
await this.#deleteWorkerSessions({ userId, appUid });
}
async #deleteWorkerSessions(filter: {
workerName?: string;
userId?: number;
appUid?: string;
}): Promise<void> {
const deleted = (await this.stores.session.deleteWorkerSessions(
filter,
)) as { uuid: string; userId: number; accessTokenUid: string | null }[];
const byUser = new Map<number, string[]>();
for (const row of deleted) {
byUser.set(row.userId, [
...(byUser.get(row.userId) ?? []),
row.uuid,
]);
if (row.accessTokenUid) {
await this.#dropAccessTokenGrants(row.accessTokenUid);
}
}
for (const [userId, uuids] of byUser) {
this.#announceRevocation({ userId, uuids });
}
}
/**
* Password-reset cascade: revoke every interactive (web/app) session for
* the user, so a hijacked session doesn't outlive a password reset. No
@@ -2272,6 +2331,30 @@ export class AuthService extends PuterService {
}
}
/**
* A worker credential lives only as long as the worker it was minted for.
* Tokens minted before `worker_uid` existed are held to the worker's name
* and owner instead.
*/
async #workerStillDeployed(
decoded: {
worker?: boolean;
worker_name?: string;
worker_uid?: string;
},
userId: number,
): Promise<boolean> {
if (!decoded.worker) return true;
// Events handler sessions have no worker row; EventsService retires them.
if (decoded.worker_name === EVENTS_WORKER_SESSION_NAME) return true;
if (!decoded.worker_name) return false;
const row = await this.stores.subdomain.getBySubdomain(
`${WORKER_SUBDOMAIN_PREFIX}${decoded.worker_name}`,
);
if (!row || Number(row.user_id) !== Number(userId)) return false;
return !decoded.worker_uid || row.uuid === decoded.worker_uid;
}
async #actorFromSessionToken(
decoded: SessionTokenPayload,
ctx: AuthenticateContext = {},
@@ -2302,6 +2385,9 @@ export class AuthService extends PuterService {
const session: SessionRow | null = rawRow;
if (!session) return { invalid: true };
if (!(await this.#workerStillDeployed(decoded, user.id as number))) {
return { invalid: true };
}
void this.#touchSession(session.uuid, user.id, ctx);
@@ -2346,6 +2432,9 @@ export class AuthService extends PuterService {
if (notBefore !== null && createdAt > 0 && createdAt < notBefore) {
return { reauth: { reason: 'session_revoked' } };
}
if (!(await this.#workerStillDeployed(decoded, user.id as number))) {
return { invalid: true };
}
void this.#touchSession(session.uuid, user.id, ctx);
+10 -1
View File
@@ -53,6 +53,9 @@ export interface SessionTokenPayload extends TokenPayloadBase {
user_uid: string;
/** Set on a worker credential riding this token type; not a browser session. */
worker?: boolean;
worker_name?: string;
/** Uuid of the worker's subdomain row; the token dies with that row. */
worker_uid?: string;
}
/**
@@ -67,6 +70,10 @@ export interface AppUnderUserTokenPayload extends TokenPayloadBase {
session?: string;
/** Events handler runs behind this token's writes; see `Actor.handlerDepth`. */
handler_depth?: number;
worker?: boolean;
worker_name?: string;
/** Uuid of the worker's subdomain row; the token dies with that row. */
worker_uid?: string;
}
/**
@@ -94,7 +101,9 @@ export interface AccessTokenPayload extends TokenPayloadBase {
}
export type AnyTokenPayload =
SessionTokenPayload | AppUnderUserTokenPayload | AccessTokenPayload;
| SessionTokenPayload
| AppUnderUserTokenPayload
| AccessTokenPayload;
// -- Session row (from `sessions` table) ----------------------------
@@ -70,6 +70,13 @@ describe('addresses', () => {
'[email protected]',
]);
});
test('a username with capitals yields lowercase addresses', () => {
expect(puterEmailAddressesOf('Dan')).toEqual([
'[email protected]',
'[email protected]',
]);
});
});
describe('account eligibility', () => {
+6 -2
View File
@@ -59,9 +59,13 @@ export const isPuterEmailAddress = (address: string): boolean =>
export const puterEmailUsername = (address: string): string =>
address.split('@')[0];
/** Every address that resolves to `username`. */
/**
* Every address that resolves to `username`, lowercased: callers compare these
* against addresses parsed out of headers, which are lowercased too, and a
* username stored with capitals would otherwise match nothing.
*/
export const puterEmailAddressesOf = (username: string): string[] =>
PUTER_EMAIL_DOMAINS.map((domain) => `${username}@${domain}`);
PUTER_EMAIL_DOMAINS.map((domain) => `${username.toLowerCase()}@${domain}`);
/**
* A temp account has neither a password nor an email - the same test
@@ -21,7 +21,10 @@ import { createServer } from 'node:http';
import type { AddressInfo } from 'node:net';
import { Readable } from 'node:stream';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { SubdomainRow } from '../../stores/subdomain/SubdomainStore.js';
import {
WORKER_SUBDOMAIN_PREFIX,
type SubdomainRow,
} from '../../stores/subdomain/SubdomainStore.js';
import type { PuterServer } from '../../server.js';
import { createTestUser, setupTestServer } from '../../testUtil.js';
import { getWorkerPreamble } from '../../drivers/workers/WorkerDriver.js';
@@ -82,6 +85,18 @@ const subdomainRow = (over: Partial<SubdomainRow>): SubdomainRow =>
...over,
}) as SubdomainRow;
/** A stored worker row: the minted token is only valid while it exists. */
const seedWorkerRow = async (
workerName: string,
over: { root_dir_id: number; app_owner?: number },
): Promise<SubdomainRow> =>
(await server.stores.subdomain.create({
userId: ownerUserId,
subdomain: `${WORKER_SUBDOMAIN_PREFIX}${workerName}`,
rootDirId: over.root_dir_id,
appOwner: over.app_owner ?? null,
})) as unknown as SubdomainRow;
describe('LocalWorkerService.reconstructDeployArgs', () => {
it('mints a user-scoped worker session token and prepends the preamble', async () => {
const src = await writeSource(
@@ -91,7 +106,7 @@ describe('LocalWorkerService.reconstructDeployArgs', () => {
const [name, authorization, code] =
await localWorkers.reconstructDeployArgs(
'worker-a',
subdomainRow({ root_dir_id: src.id }),
await seedWorkerRow('worker-a', { root_dir_id: src.id }),
);
const source = "export default { fetch: () => new Response('a') };";
@@ -130,7 +145,10 @@ describe('LocalWorkerService.reconstructDeployArgs', () => {
const [, authorization] = await localWorkers.reconstructDeployArgs(
'worker-b',
subdomainRow({ root_dir_id: src.id, app_owner: app.id }),
await seedWorkerRow('worker-b', {
root_dir_id: src.id,
app_owner: app.id,
}),
);
const auth = await server.services.auth.authenticate(authorization);
@@ -268,12 +268,15 @@ export class LocalWorkerService extends PuterService {
ownerActor,
app.uid,
workerName,
{ workerUid: row.uuid ? String(row.uuid) : undefined },
);
} else {
const session = await this.services.auth.createWorkerSessionToken(
ownerActor,
ownerUser,
workerName,
{},
{ workerUid: row.uuid ? String(row.uuid) : undefined },
);
authorization = session.token;
@@ -456,6 +456,77 @@ export class SessionStore extends PuterStore {
};
}
/**
* Hard-delete the worker sessions matching every given filter, plus the
* sessions derived from them via `parent_session_id`. A deleted worker
* never comes back, so there is no revoked row worth keeping.
*
* Returns each deleted row as `{ uuid, userId, accessTokenUid }` so the
* caller can announce the revocation and drop access-token grants.
*
* @param {{
* workerName?: string;
* userId?: number;
* appUid?: string;
* }} filter
*/
async deleteWorkerSessions({ workerName, userId, appUid } = {}) {
const where = ["`kind` = 'worker'"];
const params = [];
if (workerName) {
const workerNameExpr = this.clients.db.jsonTextExtract('`meta`', [
'worker_name',
]);
where.push(`${workerNameExpr} = ?`);
params.push(workerName);
}
if (userId) {
where.push('`user_id` = ?');
params.push(userId);
}
if (appUid) {
where.push('`app_uid` = ?');
params.push(appUid);
}
// No filter would mean every worker session on the server.
if (params.length === 0) return [];
// Primary: a session minted moments ago must not survive a replica miss.
const roots = await this.clients.db.pread(
`SELECT \`uuid\` FROM \`sessions\` WHERE ${where.join(' AND ')}`,
params,
);
if (roots.length === 0) return [];
const rootUuids = roots.map((r) => String(r.uuid));
const placeholders = rootUuids.map(() => '?').join(', ');
const scope = `\`uuid\` IN (${placeholders}) OR \`parent_session_id\` IN (${placeholders})`;
const scopeParams = [...rootUuids, ...rootUuids];
const rows = await this.clients.db.pread(
'SELECT `uuid`, `user_id`, `kind`, `app_uid`, `legacy_token_uid`, `meta`, `created_via`, `last_ip`, `last_user_agent`, `access_token_uid` ' +
`FROM \`sessions\` WHERE ${scope}`,
scopeParams,
);
// Double-delete: see `removeByUuid` for rationale.
const keys = [];
for (const r of rows) keys.push(...this.#allCacheKeysForRow(r));
await this.publishCacheKeys({ keys, broadcast: true });
await this.clients.db.write(
`DELETE FROM \`sessions\` WHERE ${scope}`,
scopeParams,
);
await this.publishCacheKeys({ keys, broadcast: true });
return rows.map((r) => ({
uuid: String(r.uuid),
userId: Number(r.user_id),
accessTokenUid: r.access_token_uid
? String(r.access_token_uid)
: null,
}));
}
/**
* Access-token identities among the rows `revokeCascade(rootUuid)` would
* affect — the root itself when it is a token row, plus any token row
@@ -805,6 +805,123 @@ describe('SessionStore', () => {
});
});
describe('deleteWorkerSessions', () => {
const name = () => `wk-${Math.random().toString(36).slice(2, 8)}`;
it('deletes every row of a worker name across users and apps, and nothing else', async () => {
const alice = await makeUser();
const bob = await makeUser();
const workerName = name();
const userScoped = await target.getOrCreateWorker(alice.id, {
appUid: null,
workerName,
});
const appScoped = await target.getOrCreateWorker(bob.id, {
appUid: `app-${uuidv4()}`,
workerName,
});
const otherWorker = await target.getOrCreateWorker(alice.id, {
appUid: null,
workerName: name(),
});
const deleted = await target.deleteWorkerSessions({ workerName });
expect(deleted.map((r: { uuid: string }) => r.uuid).sort()).toEqual(
[userScoped.uuid, appScoped.uuid].sort(),
);
expect(await rawRow(userScoped.uuid)).toBeNull();
expect(await rawRow(appScoped.uuid)).toBeNull();
expect(await rawRow(otherWorker.uuid)).not.toBeNull();
});
it('filters by user and app together', async () => {
const user = await makeUser();
const other = await makeUser();
const appUid = `app-${uuidv4()}`;
const deployed = await target.getOrCreateWorker(user.id, {
appUid,
workerName: name(),
});
const otherApp = await target.getOrCreateWorker(user.id, {
appUid: `app-${uuidv4()}`,
workerName: name(),
});
const userScoped = await target.getOrCreateWorker(user.id, {
appUid: null,
workerName: name(),
});
const otherUser = await target.getOrCreateWorker(other.id, {
appUid,
workerName: name(),
});
await target.deleteWorkerSessions({ userId: user.id, appUid });
expect(await rawRow(deployed.uuid)).toBeNull();
expect(await rawRow(otherApp.uuid)).not.toBeNull();
expect(await rawRow(userScoped.uuid)).not.toBeNull();
expect(await rawRow(otherUser.uuid)).not.toBeNull();
});
it('takes derived sessions with it', async () => {
const user = await makeUser();
const worker = await target.getOrCreateWorker(user.id, {
appUid: `app-${uuidv4()}`,
workerName: name(),
});
const child = await target.create(user.id, {
kind: 'access_token',
parent_session_id: worker.uuid,
access_token_uid: uuidv4(),
});
const deleted = await target.deleteWorkerSessions({
userId: user.id,
appUid: worker.app_uid,
});
expect(await rawRow(child.uuid)).toBeNull();
expect(
deleted.find((r: { uuid: string }) => r.uuid === child.uuid)
?.accessTokenUid,
).toBe(child.access_token_uid);
});
it('drops the cached rows, so a lookup misses and a redeploy mints fresh', async () => {
const user = await makeUser();
const workerName = name();
const first = await target.getOrCreateWorker(user.id, {
appUid: null,
workerName,
});
// Warm both the uuid and the composite worker cache.
expect(await target.getByUuidAny(first.uuid)).toBeTruthy();
expect(
await target.getWorker(user.id, { appUid: null, workerName }),
).toBeTruthy();
await target.deleteWorkerSessions({ workerName });
expect(await target.getByUuidAny(first.uuid)).toBeNull();
const second = await target.getOrCreateWorker(user.id, {
appUid: null,
workerName,
});
expect(second.uuid).not.toBe(first.uuid);
});
it('refuses to run without a filter', async () => {
const user = await makeUser();
const row = await target.getOrCreateWorker(user.id, {
appUid: null,
workerName: name(),
});
expect(await target.deleteWorkerSessions({})).toEqual([]);
expect(await rawRow(row.uuid)).not.toBeNull();
});
});
describe('error propagation (no silent swallow)', () => {
// INSERT-IGNORE used to mask every constraint violation, not just
// the partial-unique-index conflict the `getOrCreate*` paths rely
@@ -510,6 +510,7 @@ export class SubdomainStore extends PuterStore {
* }} opts
*/
async create({
uuid = uuidv4(),
userId,
subdomain,
rootDirId = null,
@@ -518,6 +519,8 @@ export class SubdomainStore extends PuterStore {
preambleVersion = null,
isProtected = false,
}: {
/** Pre-generated by a caller that has to hand the uuid out first. */
uuid?: string;
userId: number;
subdomain: string;
rootDirId?: number | null;
@@ -530,7 +533,6 @@ export class SubdomainStore extends PuterStore {
if (!userId || !subdomain) {
throw new Error('create: userId and subdomain are required');
}
const uuid = uuidv4();
await this.clients.db.write(
`INSERT INTO \`subdomains\`
(\`uuid\`, \`subdomain\`, \`user_id\`, \`root_dir_id\`, \`associated_app_id\`, \`app_owner\`, \`preamble_version\`, \`protected\`)
+11 -2
View File
@@ -17,6 +17,7 @@ import {
import type { PoolConfig } from 'pg';
import { ADMIN_GROUP_UID } from './services/selfhosted/DefaultUserService';
import { FULL_API_ACCESS } from './services/permission/consts';
import { WORKER_SUBDOMAIN_PREFIX } from './stores/subdomain/SubdomainStore.js';
import { generateDefaultFsentries } from './util/userProvisioning';
export const POSTGRES_TEST_MIGRATIONS_PATH =
@@ -309,12 +310,20 @@ export const createTestUser = async (
);
// Mint a user-scoped worker token the same way deploying an app-less
// worker does (WorkerDriver falls back to createWorkerSessionToken).
// worker does (WorkerDriver falls back to createWorkerSessionToken). The
// token only lives as long as its worker row, so seed one.
const workerName = `test-env-worker-${user.id}`;
const workerRow = await server.stores.subdomain.create({
userId: user.id,
subdomain: `${WORKER_SUBDOMAIN_PREFIX}${workerName}`,
});
const { token: workerToken } =
await server.services.auth.createWorkerSessionToken(
makeActor({ user }),
user,
'puter-test-env-worker',
workerName,
{},
{ workerUid: workerRow.uuid },
);
return {
+12 -8
View File
@@ -42,8 +42,8 @@ A plain-text body. Shorthand for `text`; use the options form for anything else.
- `attachments` (Array) - Up to **10** attachments, **25 MB** in total. Each is an object:
- `filename` (String) - Required with `content`; defaults to the file's name for a `path`/`uid` attachment.
- `content` (String) - The file body as base64. Mutually exclusive with `path`/`uid`.
- `path` (String) - A Puter path (`~/` allowed). The file is read on the server with the caller's file permissions, falling back to the worker owner's, and streamed rather than uploaded — prefer this over `content` for anything larger than a few hundred kilobytes.
- `uid` (String) - A Puter file uid, as an alternative to `path`.
- `path` (String) - A Puter path (`~/` allowed, meaning the caller's home). The file is read on the server with the caller's file permissions only, and streamed rather than uploaded — prefer this over `content` for anything larger than a few hundred kilobytes.
- `uid` (String) - A Puter file uid, as an alternative to `path`. Read with the caller's file permissions, falling back to the worker owner's — use a uid to attach a file from the worker owner's storage.
- `contentType` (String) - MIME type. Detected from the file for `path`/`uid` attachments.
- `cid` (String) - Content-ID that makes the attachment an inline part of the `html` body: with `cid: 'logo'`, `<img src="cid:logo">` shows it. Printable ASCII with no whitespace or angle brackets. Inline parts are billed like any other attachment.
@@ -52,7 +52,7 @@ A plain-text body. Shorthand for `text`; use the options form for anything else.
A `Promise` that resolves to an object:
- `messageId` (String | null) - The `Message-ID` of the first delivery: the transport's for a relayed copy, the stored message's for a Puter mailbox.
- `cost` (Number) - What the send was charged, in microcents.
- `cost` (Number) - What the send was charged, in microcents: one delivery's price for each recipient in neither `failed` nor `suppressed`.
- `suppressed` (Array) - Recipients (lowercased) dropped because they unsubscribed from your app's mail. The message went to the others.
- `failed` (Array) - Recipients (lowercased) whose delivery failed. Everyone else got their copy — retry with just these addresses.
@@ -65,7 +65,7 @@ In case of an error, the `Promise` rejects with `{ message, code }`. Codes you m
- `subscription_required` (402) - The calling account is not on a paid plan.
- `insufficient_funds` (402) - The calling account has no usage credit left for this send.
- `bad_request` (400) - Invalid arguments: a bad address, a missing body, too many recipients, an invalid attachment.
- `not_found` (404) - Every recipient was a Puter address with no mailbox to receive in.
- `not_found` (404) - An attachment's `path`/`uid` could not be read; or every recipient was a Puter address with no mailbox to receive in.
- `too_many_requests` (429) - Over the rate limit; see [Rate Limits and Quotas](/rate-limits-and-quotas/#email).
## Recipients on Puter
@@ -84,7 +84,7 @@ Every message carries an unsubscribe link and a report-abuse link. Opting out is
## Limits and cost
Sends are rate-limited per calling account; the numbers are on [Rate Limits and Quotas](/rate-limits-and-quotas/#email). A send costs a flat charge per message plus a per-byte charge for attachments, and is charged to the calling account. A send that reaches nobody is not charged.
Sends are rate-limited per calling account; the numbers are on [Rate Limits and Quotas](/rate-limits-and-quotas/#email). Each recipient's delivery costs a flat charge plus a per-byte charge for the attachments it carries, so a send to three recipients costs three deliveries. Only deliveries that succeed are charged — retrying the `failed` addresses never pays twice — and a send that reaches nobody is not charged. The calling account pays.
## Examples
@@ -123,6 +123,10 @@ router.post('/api/receipt', async ({ request, user }) => {
<strong class="example-title">Attach a file and show an inline image</strong>
```js
// The uid of a file in the worker owner's storage, e.g. from
// `(await me.puter.fs.stat('~/Public/logo.png')).uid`.
const LOGO_UID = '...';
router.post('/api/invoice', async ({ request, user }) => {
const { to } = await request.json();
return await user.puter.email.sendTransactional({
@@ -130,9 +134,9 @@ router.post('/api/invoice', async ({ request, user }) => {
subject: 'Your invoice',
html: '<img src="cid:logo" alt="Acme"><p>Your invoice is attached.</p>',
attachments: [
// Streamed from the worker owner's storage; shown inline via the cid.
{ path: '~/Public/logo.png', cid: 'logo' },
// Streamed from the user's own storage as a regular attachment.
// A uid falls back to the worker owner's storage; shown inline via the cid.
{ uid: LOGO_UID, cid: 'logo' },
// A path is always the caller's: streamed from the user's own storage.
{ path: '~/Documents/invoice.pdf' },
],
emailAccessToken: me.puter.authToken,
+4 -3
View File
@@ -24,9 +24,10 @@ import { forgetGodmodeTokens } from './godmodeTokens.js';
* response.
*
* `app` rows, plus the `access_token` row a godmode app runs on (the only
* access tokens that carry an `app_uid`). `worker` rows also carry one but are
* deployment credentials rather than this user's grant to the app, and access
* tokens the app issued follow on their own through the server-side cascade.
* access tokens that carry an `app_uid`). `worker` rows also carry one, but the
* server deletes them itself when the app's grants are revoked with `*`, and
* access tokens the app issued follow on their own through the server-side
* cascade.
*
* @param {{ uuid?: string; kind?: string; app_uid?: string | null }[]} sessions
* @param {string} appUid
+116 -30
View File
@@ -2,18 +2,23 @@
// Multipart email files are handled as String'd FormData to save on Multipart parsing logic. The primary goal of this library
// is not necessarily high performance but high maintanability.
import { PuterJSError } from '../../lib/PuterJSError.js';
/** Every refusal here is the caller's input; the SDK error contract wants a code. */
const invalid = (message) => new PuterJSError(message, 'invalid_request');
/**
* One attachment: either inline base64 `content`, or a Puter FS reference
* (`path`/`uid`) read server-side with the caller's — falling back to the
* authorizing worker's — file permissions. FS references are streamed from
* (`path`/`uid`) read server-side with the caller's file permissions; a `uid`
* falls back to the authorizing worker's. FS references are streamed from
* storage and never travel through the request, so prefer them for anything
* larger than a few hundred kilobytes.
*
* @typedef {Object} EmailAttachment
* @property {string} [filename] Required with `content`; defaults to the file's name for FS refs.
* @property {string} [content] Base64 file body. Mutually exclusive with `path`/`uid`.
* @property {string} [path] Puter FS path (supports `~/`). Mutually exclusive with `content`.
* @property {string} [uid] Puter FS entry uid. Mutually exclusive with `content`.
* @property {string} [path] Puter FS path (supports `~/`, the caller's home). Mutually exclusive with `content`.
* @property {string} [uid] Puter FS entry uid; the only way to attach a file of the worker owner's. Mutually exclusive with `content`.
* @property {string|undefined} [cid] Email content ID for inline images
* @property {string} [contentType] MIME type of the attachment.
*/
@@ -40,25 +45,103 @@ export function getRFC822DateUTC(date = new Date()) {
export function emlHeader(key, value) {
if (!isASCII(key)) {
throw new Error("eml header key " + key + " is not valid ASCII");
throw invalid("eml header key " + key + " is not valid ASCII");
}
if (Array.isArray(value)) {
value.forEach((val) => {
if (!isASCII(val)) {
throw new Error("eml header value " + val + " is not valid ASCII");
throw invalid("eml header value " + val + " is not valid ASCII");
}
})
return `${key}: ${value.join(', ')}\r\n`;
} else {
if (!isASCII(value)) {
throw new Error("eml header value " + value + " is not valid ASCII");
throw invalid("eml header value " + value + " is not valid ASCII");
}
return `${key}: ${value}\r\n`;
}
}
// UTF-8 bytes per encoded word: 45 bytes is 60 base64 characters, which with
// the `=?UTF-8?B?` and `?=` wrapping stays inside RFC 2047's 75.
const ENCODED_WORD_BYTES = 45;
const bytesToBase64 = (bytes) => {
let binary = '';
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
return btoa(binary);
};
/**
* Header text as RFC 2047 encoded words: printable ASCII comes back as is,
* anything else as `=?UTF-8?B?...?=` words split on character boundaries and
* folded onto continuation lines. Control characters are refused either way:
* they have no business in a header, encoded or not.
*
* @param {string} text
* @returns {string}
*/
export function encodeWord(text) {
if (typeof text !== 'string' || /[\x00-\x1f\x7f]/.test(text)) {
throw invalid("eml header value " + text + " is not valid");
}
if (isASCII(text)) return text;
const encoder = new TextEncoder();
const words = [];
let pending = [];
for (const char of text) {
const bytes = encoder.encode(char);
if (pending.length + bytes.length > ENCODED_WORD_BYTES) {
words.push(pending);
pending = [];
}
pending.push(...bytes);
}
if (pending.length > 0) words.push(pending);
return words.map(word => `=?UTF-8?B?${bytesToBase64(word)}?=`).join('\r\n ');
}
/**
* One address-list entry, its display name encoded when it needs to be. The
* address itself has to stay ASCII: mail without SMTPUTF8 has no encoding
* for it.
*
* @param {string} entry `addr@host` or `Name <addr@host>`
* @returns {string}
*/
function encodeAddress(entry) {
if (isASCII(entry)) return entry;
const match = /^\s*(.*?)\s*<([^<>]*)>\s*$/.exec(String(entry));
if (!match || !isASCII(match[2])) {
throw invalid("eml address " + entry + " is not valid ASCII");
}
const name = match[1].replace(/^"(.*)"$/, '$1').replace(/\\(.)/g, '$1');
return `${encodeWord(name)} <${match[2]}>`;
}
/** A free-text header (Subject), RFC 2047 encoded when not plain ASCII. */
export function textHeader(key, value) {
if (!isASCII(key)) {
throw invalid("eml header key " + key + " is not valid ASCII");
}
return `${key}: ${encodeWord(value)}\r\n`;
}
/**
* An address-list header (To, Cc, Bcc, Reply-To, From) with non-ASCII display
* names encoded. A string is one entry, or an ASCII list passed through as is.
*/
export function addressHeader(key, value) {
if (!isASCII(key)) {
throw invalid("eml header key " + key + " is not valid ASCII");
}
const entries = Array.isArray(value) ? value : [value];
return `${key}: ${entries.map(encodeAddress).join(', ')}\r\n`;
}
export function determineTopLevelMimeType({ attachments = [], text, html }) {
const inlineAttachments = attachments.filter(a => a.cid);
const regularAttachments = attachments.filter(a => !a.cid);
@@ -78,7 +161,7 @@ export function determineTopLevelMimeType({ attachments = [], text, html }) {
if (html) {
return { isMultiPart: false, mimeType: 'text/html; charset=UTF-8' };
}
throw new Error('Email must have text, html, or an attachment');
throw invalid('Email must have text, html, or an attachment');
}
@@ -100,21 +183,22 @@ export const isASCII = (str) => /^[\x20-\x7E]*$/.test(str);
/**
* Composes a multipart email attachment
*
* @param {EmailAttachment} attachment
*
* @param {EmailAttachment} attachment
* @param {import('../../index.js').Puter} [puter] Reads `path` attachments; the module's own instance.
*/
export async function composeAttachment({ cid, content, contentType, filename, path, uid }) {
export async function composeAttachment({ cid, content, contentType, filename, path, uid }, puter = globalThis.puter) {
if (path && content) {
throw new Error("Path and Content are mutually exclusive");
throw invalid("Path and Content are mutually exclusive");
}
if (!path && !content && !uid) {
throw new Error("Attachment requires content, path, or uid");
throw invalid("Attachment requires content, path, or uid");
}
if (uid) {
throw new Error("uid attachments are not supported yet");
throw invalid("uid attachments are not supported yet");
}
if (contentType && !filename) {
throw new Error("filename parameter required for contentType");
throw invalid("filename parameter required for contentType");
}
let b64emailContent = content;
@@ -129,12 +213,12 @@ export async function composeAttachment({ cid, content, contentType, filename, p
}
if (!filename) {
throw new Error("Attachment's filename cannot be determined automatically. Please provide filename!");
throw invalid("Attachment's filename cannot be determined automatically. Please provide filename!");
}
filename = filename.replaceAll('\\', '\\\\');
filename = filename.replaceAll('"', '\\"');
if (!isASCII(filename)) {
throw new Error("filename must be ascii");
throw invalid("filename must be ascii");
}
if (contentType) {
@@ -142,7 +226,7 @@ export async function composeAttachment({ cid, content, contentType, filename, p
}
if (!mimeType) {
throw new Error("Attachment's mimetype cannot be determined automatically. Please provide contentType!");
throw invalid("Attachment's mimetype cannot be determined automatically. Please provide contentType!");
}
mimeType += ';name="' + filename + '"';
@@ -168,34 +252,36 @@ export function combineParts(parts, boundary) {
}
/**
*
* @param {EmailComposeOptions} param0
*
* @param {EmailComposeOptions} param0
* @param {import('../../index.js').Puter} [puter] The module's own instance: who the mail is from, and
* whose storage `path` attachments are read from.
*/
export async function compose({ from, to, cc, bcc, subject, replyTo, attachments = [], text, html }) {
export async function compose({ from, to, cc, bcc, subject, replyTo, attachments = [], text, html }, puter = globalThis.puter) {
// --- Header constructing logic ---
let headerLines = "MIME-Version: 1.0\r\n";
headerLines += emlHeader('Date', getRFC822DateUTC());
if (subject) {
headerLines += emlHeader('Subject', subject);
headerLines += textHeader('Subject', subject);
}
if (to) {
headerLines += emlHeader('To', to);
headerLines += addressHeader('To', to);
} else {
throw new Error("Cannot compose email without recipient!");
throw invalid("Cannot compose email without recipient!");
}
if (cc) {
headerLines += emlHeader('cc', cc);
headerLines += addressHeader('cc', cc);
}
if (bcc) {
headerLines += emlHeader('bcc', bcc);
headerLines += addressHeader('bcc', bcc);
}
if (replyTo) {
headerLines += emlHeader('Reply-To', replyTo);
headerLines += addressHeader('Reply-To', replyTo);
}
if (from) {
headerLines += emlHeader('From', from);
headerLines += addressHeader('From', from);
} else {
// Add From Block (hardcoded to puter.email for now)
const userinfo = await puter.getUser();
@@ -229,8 +315,8 @@ export async function compose({ from, to, cc, bcc, subject, replyTo, attachments
const inlineAttachments = attachments.filter(a => a.cid);
const regularAttachments = attachments.filter(a => !a.cid);
const inlineParts = await Promise.all(inlineAttachments.map(composeAttachment));
const regularParts = await Promise.all(regularAttachments.map(composeAttachment));
const inlineParts = await Promise.all(inlineAttachments.map(a => composeAttachment(a, puter)));
const regularParts = await Promise.all(regularAttachments.map(a => composeAttachment(a, puter)));
// Build the text/html content as one part. Only wrap it in its own nested
// multipart/alternative (fresh boundary) when both text and html are present.
@@ -502,8 +502,63 @@ describe('compose', () => {
await expect(compose({ ...baseOptions, attachments: [] })).rejects.toThrow(/text, html, or an attachment/);
});
it('rejects a non-ASCII subject', async () => {
await expect(compose({ ...baseOptions, subject: 'héllo', text: 'hi' })).rejects.toThrow(/not valid ASCII/);
it('encodes a non-ASCII subject as RFC 2047 words instead of refusing it', async () => {
const eml = await compose({ ...baseOptions, subject: 'héllo', text: 'hi' });
expect(eml).toContain('Subject: =?UTF-8?B?aMOpbGxv?=\r\n');
expect(isASCII(eml.split('\r\n\r\n')[0].replace(/\r\n/g, ''))).toBe(true);
});
it('folds a long non-ASCII subject into words of at most 75 characters', async () => {
const subject = 'Réunion — 東京 🎉 '.repeat(12);
const eml = await compose({ ...baseOptions, subject, text: 'hi' });
const head = eml.split('\r\n\r\n')[0];
const subjectLines = head.slice(head.indexOf('Subject: ')).split('\r\n');
const words = [subjectLines[0].slice('Subject: '.length)];
for ( const line of subjectLines.slice(1) ) {
if ( ! line.startsWith(' ') ) break;
words.push(line.slice(1));
}
expect(words.length).toBeGreaterThan(1);
for ( const word of words ) {
expect(word).toMatch(/^=\?UTF-8\?B\?[A-Za-z0-9+/=]+\?=$/);
expect(word.length).toBeLessThanOrEqual(75);
}
});
it('refuses control characters in a subject, encoded or not', async () => {
await expect(compose({ ...baseOptions, subject: 'hi\r\nBcc: [email protected]', text: 'hi' }))
.rejects.toMatchObject({ code: 'invalid_request' });
await expect(compose({ ...baseOptions, subject: 'héllo\r\nBcc: [email protected]', text: 'hi' }))
.rejects.toMatchObject({ code: 'invalid_request' });
});
it('refuses a non-ASCII address while allowing a non-ASCII display name', async () => {
await expect(compose({ ...baseOptions, to: 'ü@example.com', text: 'hi' }))
.rejects.toMatchObject({ code: 'invalid_request' });
await expect(compose({ ...baseOptions, to: 'Jürgen <ü@example.com>', text: 'hi' }))
.rejects.toMatchObject({ code: 'invalid_request' });
await expect(compose({ ...baseOptions, to: 'Jürgen <[email protected]>', text: 'hi' })).resolves.toBeTypeOf('string');
});
it('rejects with a coded PuterJSError, per the SDK error contract', async () => {
await expect(compose({ subject: 'hi', text: 'hi' }))
.rejects.toMatchObject({ name: 'PuterJSError', code: 'invalid_request' });
await expect(composeAttachment({ uid: 'uid-1' }))
.rejects.toMatchObject({ code: 'invalid_request' });
expect(() => emlHeader('Subject', 'héllo')).toThrow(expect.objectContaining({ code: 'invalid_request' }));
});
it('uses the instance it is given rather than the global one', async () => {
const own = {
fs: { read: vi.fn(async () => new Blob([PNG_BYTES], { type: 'image/png' })) },
getUser: vi.fn(async () => ({ username: 'carol' })),
};
const eml = await compose({ to: '[email protected]', subject: 'hi', html: '<p/>', attachments: [{ path: '~/x.png', cid: 'c' }] }, own);
expect(eml).toContain('From: [email protected]\r\n');
expect(own.fs.read).toHaveBeenCalledWith('~/x.png');
expect(own.getUser).toHaveBeenCalled();
expect(fsRead).not.toHaveBeenCalled();
expect(getUser).not.toHaveBeenCalled();
});
});
@@ -513,6 +568,24 @@ describe('compose round-trips through a MIME parser', () => {
// The parser terminates every text body with a newline of its own.
const bodyOf = (str) => str?.replace(/\n$/, '');
it('recovers a UTF-8 subject and display names', async () => {
const subject = 'Grüße ✓ — 東京 🎉 '.repeat(6).trim();
const parsed = await parse({
subject,
to: ['"José Pérez" <[email protected]>', '[email protected]'],
cc: 'Zoë <[email protected]>',
from: 'Ålice <[email protected]>',
text: 'hi',
});
expect(parsed.subject).toBe(subject);
expect(parsed.to).toEqual([
{ address: '[email protected]', name: 'José Pérez' },
{ address: '[email protected]', name: '' },
]);
expect(parsed.cc).toEqual([{ address: '[email protected]', name: 'Zoë' }]);
expect(parsed.from).toEqual({ address: '[email protected]', name: 'Ålice' });
});
it('recovers a plain text body', async () => {
const parsed = await parse({ text: 'hello there' });
expect(bodyOf(parsed.text)).toBe('hello there');
+16 -3
View File
@@ -32,7 +32,8 @@ import { list } from './list.js';
*
* @typedef {Object} EmailSendResult
* @property {string | null} messageId First transport message id reported for this send, when available.
* @property {number} cost Total charge for this send, in microcents.
* @property {number} cost Total charge for this send, in microcents: one delivery's price for each
* recipient that got a copy.
* @property {string[]} suppressed Recipients omitted because they opted out of this app's mail.
* @property {string[]} failed Recipients whose delivery attempt failed. Everyone else got their copy —
* retry with just these addresses. A send where every delivery fails rejects instead.
@@ -82,7 +83,8 @@ const preprocessSendArgs = (args) => {
* // Inline or Puter-FS attachments:
* attachments: [
* { filename, content, contentType }, // content = base64
* { path: '~/Documents/report.pdf' }, // streamed server-side
* { path: '~/Documents/report.pdf' }, // the caller's file, streamed server-side
* { uid: LOGO_UID }, // a uid may also name a file of the worker owner's
* ],
* });
* });
@@ -141,6 +143,9 @@ export class EmailModule extends PuterModule {
iface: 'puter-transactional-email',
method: 'sendTransactional',
argNames: ['to', 'subject', 'body'],
// This module's instance, not `globalThis.puter`: in a worker the two
// differ (`user.puter` vs `me.puter`), and the caller is the one billed.
puter: this.puter,
preprocess: preprocessSendArgs,
upgradePrompt: {
method: 'puter.email.sendTransactional',
@@ -157,7 +162,15 @@ export class EmailModule extends PuterModule {
* @returns {Promise<unknown>}
*/
send = async (options) => {
const req = await fetchUrl(`${this.APIOrigin}/email/send`, { method: "POST", includePuterAuth: true, body: new Blob([await compose(options)], { type: 'message/rfc822' }) });
// `includePuterAuth` reads the global instance's token, so any other
// instance (a worker's `user.puter`) sends its own.
const isGlobal = this.puter === globalThis.puter;
const req = await fetchUrl(`${this.APIOrigin}/email/send`, {
method: "POST",
includePuterAuth: isGlobal,
...(isGlobal ? {} : { authToken: this.authToken }),
body: new Blob([await compose(options, this.puter)], { type: 'message/rfc822' }),
});
const result = await req.json();
if ( ! req.ok ) {
promptIfUpgradeRequired(
@@ -0,0 +1,77 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
/**
* The email methods have to act for the instance they hang off, not for
* `globalThis.puter`: in a worker, `user.puter` and `me.puter` are different
* identities, and the one making the call is the one billed and rate-limited.
*/
const captured = vi.hoisted(() => ({ specs: [], fetches: [] }));
vi.mock('../../lib/utils.js', async (importOriginal) => ({
...(await importOriginal()),
makeDriverMethod: (spec) => {
captured.specs.push(spec);
return async () => ({});
},
}));
vi.mock('../../lib/networkUtils.js', async (importOriginal) => ({
...(await importOriginal()),
fetchUrl: async (url, opts) => {
captured.fetches.push({ url, opts });
return { ok: true, status: 200, json: async () => ({ delivered: [] }) };
},
}));
const { EmailModule } = await import('./Email.js');
const origPuter = globalThis.puter;
const makePuter = (username, authToken) => ({
APIOrigin: 'https://api.example.test',
authToken,
getUser: vi.fn(async () => ({ username })),
fs: { read: vi.fn() },
});
beforeEach(() => {
captured.specs.length = 0;
captured.fetches.length = 0;
globalThis.puter = makePuter('global', 'global-token');
});
afterEach(() => {
globalThis.puter = origPuter;
});
describe('EmailModule binds to its own instance', () => {
it('hands sendTransactional its own puter, not the global one', () => {
const own = makePuter('caller', 'caller-token');
new EmailModule(own);
const spec = captured.specs.find(s => s.method === 'sendTransactional');
expect(spec.puter).toBe(own);
});
it('sends from, and authenticates as, a non-global instance', async () => {
const own = makePuter('caller', 'caller-token');
const email = new EmailModule(own);
await email.send({ to: '[email protected]', subject: 'hi', text: 'hi' });
expect(own.getUser).toHaveBeenCalled();
expect(globalThis.puter.getUser).not.toHaveBeenCalled();
const [{ url, opts }] = captured.fetches;
expect(url).toBe('https://api.example.test/email/send');
expect(opts.includePuterAuth).toBe(false);
expect(opts.authToken).toBe('caller-token');
expect(await opts.body.text()).toContain('From: [email protected]\r\n');
});
it('keeps the live-token path for the global instance', async () => {
const email = new EmailModule(globalThis.puter);
await email.send({ to: '[email protected]', subject: 'hi', text: 'hi' });
const [{ opts }] = captured.fetches;
expect(opts.includePuterAuth).toBe(true);
expect(opts.authToken).toBeUndefined();
});
});
+20 -6
View File
@@ -122,15 +122,24 @@ export const toSummary = (entry, folder) => {
};
/**
* Where a listing resumes: the day folder being walked and, while it still has
* pages, the readdir cursor within it.
* Where a listing resumes: the day folder being walked; while it still has
* pages, the readdir cursor within it; and the readdir cursor of the page of
* day folders that day was read from, so resuming does not re-list every day
* newer than it. A cursor without that last part (issued before it existed)
* walks the days from the top, which is only slower.
*
* @typedef {{ folder: EmailFolder, day: string, fsCursor?: string }} ListPosition
* @typedef {{ folder: EmailFolder, day: string, fsCursor?: string, daysCursor?: string }} ListPosition
*/
/** @param {ListPosition} position @returns {string} */
export const encodeCursor = ({ folder, day, fsCursor }) => {
const payload = { v: 1, f: folder, d: day, ...(fsCursor ? { c: fsCursor } : {}) };
export const encodeCursor = ({ folder, day, fsCursor, daysCursor }) => {
const payload = {
v: 1,
f: folder,
d: day,
...(fsCursor ? { c: fsCursor } : {}),
...(daysCursor ? { dc: daysCursor } : {}),
};
return btoa(JSON.stringify(payload));
};
@@ -153,7 +162,12 @@ export const decodeCursor = (cursor, folder) => {
if ( payload.f !== folder ) {
throw new PuterJSError('Cursor does not match the requested folder', 'invalid_request');
}
return { folder, day: payload.d, ...(typeof payload.c === 'string' ? { fsCursor: payload.c } : {}) };
return {
folder,
day: payload.d,
...(typeof payload.c === 'string' ? { fsCursor: payload.c } : {}),
...(typeof payload.dc === 'string' ? { daysCursor: payload.dc } : {}),
};
};
/** A readdir on a folder that was never created. @param {unknown} e */
+48 -32
View File
@@ -12,36 +12,49 @@ import {
/** @typedef {import('../../lib/types.js').ListPage<EmailSummary>} EmailListPage */
/** @typedef {import('./lib/mailbox.js').ListPosition} ListPosition */
// Page size for the SDK's own walks over the mailbox's day folders.
const DIRECTORY_PAGE_LIMIT = 1000;
// Day folders read per directory page. Small on purpose: a listing resumes
// from the page its cursor's day came from, so this bounds what every later
// page re-reads before reaching its own day. About a month of a mailbox that
// gets mail daily.
const DAY_PAGE_LIMIT = 30;
/**
* Every day folder in `folder`, newest first. A mailbox that was never
* written to has no folder at all, which lists as empty.
* The day folders in `folder`, newest first, read a directory page at a time
* starting at `daysCursor`. Each day comes with the cursor of the page it was
* read from, which is where a later listing can pick the walk back up. A
* mailbox that was never written to has no folder at all, which walks as
* empty.
*
* @this {import('./Email.js').EmailModule}
* @param {import('../../index.js').Puter} puter
* @param {EmailFolder} folder
* @returns {Promise<string[]>}
* @param {string | undefined} daysCursor
* @returns {AsyncGenerator<{ day: string, daysCursor?: string }>}
*/
async function listDays (folder) {
const fetchPage = pageParams => this.puter.fs.readdir({
path: folderRoot(folder),
limit: DIRECTORY_PAGE_LIMIT,
sortBy: 'name',
sortOrder: 'desc',
...pageParams,
});
const days = [];
try {
for await ( const page of iteratePages(fetchPage) ) {
for ( const entry of page.items ) {
if ( entry.is_dir && isDayName(entry.name) ) days.push(entry.name);
async function* walkDays (puter, folder, daysCursor) {
let cursor = daysCursor ?? null;
while ( true ) {
/** @type {{ items: Array<Record<string, any>>, cursor?: string }} */
let page;
try {
page = await puter.fs.readdir({
path: folderRoot(folder),
limit: DAY_PAGE_LIMIT,
sortBy: 'name',
sortOrder: 'desc',
cursor,
});
} catch (e) {
if ( ! isMissingDirectory(e) ) throw e;
return;
}
for ( const entry of page.items ) {
if ( entry.is_dir && isDayName(entry.name) ) {
yield { day: entry.name, ...(cursor ? { daysCursor: cursor } : {}) };
}
}
} catch (e) {
if ( ! isMissingDirectory(e) ) throw e;
if ( ! page.cursor ) return;
cursor = page.cursor;
}
return days.sort().reverse();
}
/**
@@ -56,18 +69,20 @@ async function listDays (folder) {
* @returns {Promise<EmailListPage>}
*/
async function fetchPage (folder, limit, position) {
const days = await listDays.call(this, folder);
const days = walkDays(this.puter, folder, position?.daysCursor);
/** @type {EmailSummary[]} */
const items = [];
// Resume at the cursor's day; a day folder deleted since the cursor was
// issued means continuing from the next older one.
let dayIndex = position ? days.findIndex(day => day <= position.day) : 0;
if ( dayIndex === -1 ) return { items };
let fsCursor = position && days[dayIndex] === position.day ? position.fsCursor : undefined;
let next = await days.next();
while ( position && ! next.done && next.value.day > position.day ) {
next = await days.next();
}
let fsCursor = position && ! next.done && next.value.day === position.day ? position.fsCursor : undefined;
while ( dayIndex < days.length && items.length < limit ) {
const day = days[dayIndex];
while ( ! next.done && items.length < limit ) {
const { day, daysCursor } = next.value;
// Keep reading this day until the page is full or the day is spent:
// entries that are not mail objects are skipped, and skipping must
// not shorten the page while the same day still has messages.
@@ -94,13 +109,14 @@ async function fetchPage (folder, limit, position) {
if ( ! fsCursor ) break;
}
if ( fsCursor ) {
return { items, cursor: encodeCursor({ folder, day, fsCursor }) };
return { items, cursor: encodeCursor({ folder, day, fsCursor, daysCursor }) };
}
dayIndex += 1;
next = await days.next();
}
if ( dayIndex < days.length ) {
return { items, cursor: encodeCursor({ folder, day: days[dayIndex] }) };
if ( ! next.done ) {
const { day, daysCursor } = next.value;
return { items, cursor: encodeCursor({ folder, day, daysCursor }) };
}
return { items };
}
@@ -79,6 +79,13 @@ describe('cursors', () => {
expect(decodeCursor(encodeCursor({ folder: 'sent', day: '2024-03-15' }), 'sent')).toEqual({ folder: 'sent', day: '2024-03-15' });
});
it('carries the day-listing cursor, and reads one issued without it', () => {
const position = { folder: 'inbox', day: '2024-03-15', fsCursor: 'abc', daysCursor: 'dc1' };
expect(decodeCursor(encodeCursor(position), 'inbox')).toEqual(position);
const legacy = btoa(JSON.stringify({ v: 1, f: 'inbox', d: '2024-03-15', c: 'abc' }));
expect(decodeCursor(legacy, 'inbox')).toEqual({ folder: 'inbox', day: '2024-03-15', fsCursor: 'abc' });
});
it('treats null and undefined as the first page', () => {
expect(decodeCursor(null, 'inbox')).toBeUndefined();
expect(decodeCursor(undefined, 'inbox')).toBeUndefined();
@@ -208,6 +215,61 @@ describe('list()', () => {
expect(calls.every(c => c.path.startsWith('~/.mail/sent'))).toBe(true);
});
it('resumes the day walk where the cursor left it instead of re-listing from the top', async () => {
// 100 days, one message each: far more day folders than one page of them.
const tree = { '~/.mail/objects': [] };
const dayIds = [];
for ( let i = 0; i < 100; i++ ) {
const ms = Date.UTC(2024, 0, 1) + i * 24 * 60 * 60 * 1000;
const day = new Date(ms).toISOString().slice(0, 10);
const id = uuidAt(ms + 1000);
tree['~/.mail/objects'].push(day);
tree[`~/.mail/objects/${ day }`] = [`${ id }--${ toB64Url(day) }`];
dayIds.unshift(id);
}
const { fs, calls } = fakeFs(tree);
const email = moduleOver(fs);
const seen = [];
let cursor = null;
do {
const before = calls.length;
const page = await email.list.call(email, { limit: 7, cursor });
seen.push(...page.items.map(m => m.id));
const dayListings = calls.slice(before).filter(c => c.path === '~/.mail/objects');
// Never more than the page holding the cursor's day plus the one
// after it, however deep into the mailbox the cursor points.
expect(dayListings.length).toBeLessThanOrEqual(2);
if ( cursor ) {
const resumedFrom = decodeCursor(cursor, 'inbox').daysCursor ?? null;
expect(dayListings[0].cursor).toBe(resumedFrom);
}
cursor = page.cursor;
} while ( cursor );
expect(seen).toEqual(dayIds);
// Only pages whose day sits in the first page of day folders start
// there; the deeper ones resumed mid-folder.
const fromTop = calls.filter(c => c.path === '~/.mail/objects' && ! c.cursor);
expect(fromTop.length).toBeLessThanOrEqual(Math.ceil(30 / 7) + 1);
expect(calls.some(c => c.path === '~/.mail/objects' && c.cursor)).toBe(true);
});
it('resumes from a cursor issued before day-listing cursors existed', async () => {
const { fs } = fakeFs(inboxTree);
const email = moduleOver(fs);
const legacy = btoa(JSON.stringify({ v: 1, f: 'inbox', d: '2024-03-15' }));
const page = await email.list.call(email, { limit: 10, cursor: legacy });
expect(page.items.map(m => m.id)).toEqual([ids.d15a, ids.d14b, ids.d14a]);
});
it('continues from the next older day when the cursor day was deleted', async () => {
const { fs } = fakeFs({ ...inboxTree, '~/.mail/objects': ['2024-03-14', '2024-03-16'] });
const email = moduleOver(fs);
const gone = encodeCursor({ folder: 'inbox', day: '2024-03-15', fsCursor: '5' });
const page = await email.list.call(email, { limit: 10, cursor: gone });
expect(page.items.map(m => m.id)).toEqual([ids.d14b, ids.d14a]);
});
it('rejects offset, includeTotal, and a foreign cursor before any request', async () => {
const { fs, calls } = fakeFs(inboxTree);
const email = moduleOver(fs);