Files

234 lines
18 KiB
JavaScript

// LLP 1116 D4: every verb of an app's exact.mjs answers `--help` at once and starts nothing; the scripts it
// runs answer `--help` and refuse a flag they do not take; and each verb's help names exactly the flags its
// script parses (so it never names one that does not exist). `bun test ./scripts/help.test.mjs`.
import { test } from 'bun:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resolve } from 'node:path';
import { createApp } from '../game/new.mjs';
import { appVerbs } from './app-cli.mjs';
import { HELP, appleBuildRefusal, guardFlags, helpText, indexText, knownFlags } from './help.mjs';
import { scriptFlags } from './help-parity.mjs';
const ROOT = resolve(import.meta.dir, '..');
const APP_VERBS = ['web', 'web-build', 'serve', 'test', 'agent', 'verify', 'ios', 'mac', 'share', 'linux', 'windows', 'android', 'update', 'lock', 'add', 'contract', 'forma', 'hatch', 'snapback', 'backend', 'pattern', 'docs', 'feedback'];
const timed = (args, options) => {
const t = performance.now(), r = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 15_000, ...options });
return { ...r, ms: performance.now() - t };
};
test('each verb of a new app\'s exact.mjs, and no verb, answers help from exact2 and runs nothing', () => {
const parent = mkdtempSync(resolve(tmpdir(), 'exact-help-'));
try {
const dir = resolve(parent, 'field-log'), sdk = resolve(parent, 'sdk'), started = resolve(parent, 'started');
createApp(dir);
const manifest = JSON.parse(readFileSync(resolve(dir, 'app.json'), 'utf8'));
writeFileSync(resolve(dir, 'app.json'), JSON.stringify({ ...manifest, commands: { seed: ['bun', 'seed.mjs'] } }, null, 2));
// A fake exact2 whose every script, run, leaves a mark: a help that built or served anything would.
mkdirSync(resolve(sdk, 'scripts'), { recursive: true });
mkdirSync(resolve(sdk, 'js/bake/src'), { recursive: true });
for (const file of ['scripts/help.mjs', 'scripts/app-cli.mjs', 'scripts/bun.mjs', 'js/bake/src/__exact_env.mjs']) copyFileSync(resolve(ROOT, file), resolve(sdk, file));
for (const file of ['host/web/dev.mjs', 'host/web/build.mjs', 'host/web/serve.mjs', 'host/apple/build.mjs', 'scripts/agent.mjs', 'scripts/verify.mjs', 'scripts/build-linux.mjs', 'scripts/agent-android.mjs', 'scripts/exact.mjs', 'scripts/feedback.mjs', 'scripts/share.mjs']) {
mkdirSync(resolve(sdk, file, '..'), { recursive: true });
writeFileSync(resolve(sdk, file), `require('node:fs').appendFileSync(${JSON.stringify(started)}, ${JSON.stringify(file)} + '\\n');`);
}
const exact = (...args) => timed([resolve(dir, 'exact.mjs'), ...args], { cwd: dir, env: { ...process.env, EXACT: sdk } });
for (const verb of APP_VERBS) {
const lead = verb === 'test' || verb === 'verify' ? ['ios'] : verb === 'agent' ? ['web', 'tree'] : [];
for (const args of [[verb, '--help'], [verb, ...lead, '-h'], ['help', verb], [verb, '--help', 'all']]) {
const r = exact(...args);
assert.equal(r.status, 0, `${args.join(' ')}: ${r.stderr}`);
assert.ok(r.stdout.startsWith(HELP[verb].usage), `${args.join(' ')} printed:\n${r.stdout}`);
assert.ok(r.ms < 2000, `${args.join(' ')} took ${Math.round(r.ms)} ms`);
assert.equal(r.stdout.trim(), helpText(verb, { all: args.includes('all') }), `${args.join(' ')}: the short text, or with all the whole`);
}
}
for (const args of [[], ['help'], ['--help'], ['-h']]) {
const r = exact(...args);
assert.equal(r.status, 0, r.stderr);
for (const verb of [...APP_VERBS, 'seed']) assert.match(r.stdout, new RegExp(`^ ${verb} +\\S`, 'm'), `the index names ${verb}`);
assert.match(r.stdout, /seed +app\.json commands: bun seed\.mjs/);
}
// LLP 1116.010 D2 (weather, round 12: `help --help all` and `--help all` reported "no verb all"): every help form the index and
// the helps name runs as written, and `all` with no verb is the index, never a verb.
const own = { seed: ['bun', 'seed.mjs'] }, index = indexText({ verbs: APP_VERBS, own }), forms = new Set();
for (const text of [index, ...APP_VERBS.map((v) => helpText(v))]) for (const [, form] of text.matchAll(/bun exact\.mjs ((?:[\w<>-]+ )?(?:--help|-h)(?: all)?|help(?: all)?)(?=[:\s]|$)/g)) forms.add(form);
assert.ok(forms.has('<verb> --help') && forms.has('<verb> --help all'), `the forms named: ${[...forms]}`);
const expected = (args) => { const verb = args.find((a) => HELP[a]); return verb ? helpText(verb, { all: args.includes('all') }) : index; };
for (const form of forms) for (const verb of form.includes('<verb>') ? APP_VERBS : ['']) {
const args = form.replace('<verb>', verb).split(' ').filter(Boolean), r = exact(...args);
assert.equal(r.status, 0, `${args.join(' ')}: ${r.stderr}`);
assert.equal(r.stdout.trim(), expected(args), `bun exact.mjs ${args.join(' ')}`);
}
for (const args of [['--help', 'all'], ['-h', 'all'], ['help', 'all'], ['help', '--help', 'all'], ['help', 'web', 'all'], ['help', 'all', 'web'], ['help', 'seed']]) {
const r = exact(...args);
assert.equal(r.status, 0, `${args.join(' ')}: ${r.stderr}`);
if (args[1] !== 'seed') assert.equal(r.stdout.trim(), expected(args), `bun exact.mjs ${args.join(' ')}`);
}
assert.equal(exact('help', 'nope').status, 2);
assert.equal(exact('help', 'all', 'nope').status, 2);
assert.ok(!existsSync(started), `help ran ${existsSync(started) && readFileSync(started, 'utf8')}`);
assert.ok(!existsSync(resolve(dir, '.exact/commands.jsonl')), 'help is not a logged command');
} finally { rmSync(parent, { recursive: true, force: true }); }
}, 60_000); // One offline Cargo resolution.
test('the scripts a verb runs answer --help and refuse a flag they do not take, before starting anything', () => {
const env = { ...process.env };
delete env.EXACT_APP_DIR;
for (const [verb, args] of [
['web', ['host/web/dev.mjs', '--app', 'caltrain']], ['web-build', ['host/web/build.mjs', 'caltrain']], ['serve', ['host/web/serve.mjs']],
['agent', ['scripts/agent.mjs', 'web', '--app', 'caltrain', 'tree']], ['test', ['scripts/agent.mjs', 'web', '--test', 'x.test.contract']],
['verify', ['scripts/verify.mjs', 'web', '--app', 'caltrain']],
['ios', ['host/apple/build.mjs', '--ios', 'caltrain-apple']], ['mac', ['host/apple/build.mjs', 'caltrain-apple']],
['linux', ['scripts/build-linux.mjs', 'caltrain']], ['android', ['scripts/agent-android.mjs', 'build', 'caltrain']],
['feedback', ['scripts/feedback.mjs', 'send']], ['hatch', ['scripts/exact.mjs', 'hatch', 'avatar']], ['add', ['scripts/exact.mjs', 'add', 'geolocation']], ['pattern', ['scripts/exact.mjs', 'pattern', 'list-detail']],
['contract', ['scripts/exact.mjs', 'contract', 'build']], ['new', ['scripts/exact.mjs', 'new', 'x']],
]) {
const help = timed([...args, '--help'], { cwd: ROOT, env });
assert.equal(help.status, 0, `${args.join(' ')} --help: ${help.stderr}`);
assert.ok(help.stdout.startsWith(HELP[verb].usage), `${args.join(' ')} --help printed:\n${help.stdout}${help.stderr}`);
assert.ok(help.ms < 2000, `${args.join(' ')} --help took ${Math.round(help.ms)} ms`);
if (verb === 'contract') continue; // the compiler parses its own flags
const refused = timed([...args, '--bogus'], { cwd: ROOT, env });
assert.equal(refused.status, 2, `${args.join(' ')} --bogus: ${refused.stdout}${refused.stderr}`);
assert.match(refused.stderr, new RegExp(`^${verb}: unknown flag --bogus\\. It takes ${HELP[verb].flags.length ? '--' : 'no flags'}`));
}
});
// A script reading `args[i + 1]` took the next flag as the value (`--archive --unsigned` wrote the archive as
// <cwd>/--unsigned) or, last, nothing and so its default (`--sim` last installed on EXACT_SIM's simulator).
test('a flag that takes a value is refused by name when the value is missing or another flag', () => {
const guarded = (verb, argv) => {
const said = [];
const code = guardFlags(verb, argv, { fail: (line) => said.push(line), exit: (code) => code });
return [code === argv ? 0 : code, said.join('\n')];
};
for (const [verb, argv, flag] of [
['ios', ['--device', '--archive'], '--archive <out.ipa>'], ['ios', ['--device', '--archive', '--unsigned'], '--archive <out.ipa>'],
['ios', ['--ios', '--sim'], '--sim <udid|name>'], ['ios', ['--device', '--phone', '--run'], '--phone <udid|name>'],
['ios', ['--ios', '--optimize'], '--optimize speed|size|balanced'], ['mac', ['hello-apple', '--optimize'], '--optimize speed|size|balanced'],
['ios', ['--ios', '--link'], '--link <plan>'], ['web-build', ['--out', '--json'], '--out <dir>'], ['agent', ['web', 'tree', '--url'], '--url <url>'],
]) assert.deepEqual(guarded(verb, argv), [2, `${verb}: ${flag.split(' ')[0]} needs a value (${flag})`], argv.join(' '));
// Every failure in one run: a missing value and an unknown flag are both said.
assert.match(guarded('ios', ['--sim', '--bogus'])[1], /^ios: --sim needs a value \(--sim <udid\|name>\)\nios: unknown flag --bogus\. /);
// A value is taken as before, a negative number among them; a value in brackets is optional.
for (const [verb, argv] of [['ios', ['--ios', '--sim', 'iPhone 17', '--optimize', 'size']], ['web', ['--port', '-1']], ['agent', ['macos', '--login']], ['agent', ['macos', '--login', 'hidden', 'tree']]]) {
assert.deepEqual(guarded(verb, argv), [0, ''], argv.join(' '));
}
assert.equal(appleBuildRefusal(['--ios', '--link', 'all']), '--link takes plan, not all');
assert.equal(appleBuildRefusal(['--ios', '--link', 'plan']), null);
});
test('forma help lists every binding language', () => {
assert.match(helpText('forma'), /gen ts\|rust\|swift\|kotlin <app>/);
});
test('each verb\'s help names exactly the flags its script parses, in a short text', () => {
assert.match(helpText('android'), /--write-locks/, 'Android authors need the lock-writing command after a graph change');
// The verbs' scripts and the flags each parses, read as caps.mjs reads them before a push (help-parity.mjs).
for (const [verb, flags] of scriptFlags(ROOT)) assert.deepEqual([...knownFlags(verb).keys()].sort(), flags, `${verb}: the help's flags (documented and internal) against its script's`);
for (const verb of Object.keys(HELP)) {
const lines = helpText(verb, { all: true }).split('\n').length;
assert.ok(lines <= 36, `${verb}'s help is ${lines} lines; it is replayed in an agent's context, so keep it short`);
}
});
test('contract help names the build and verify flags with their arguments and restrictions', () => {
const text = helpText('contract');
for (const [command, flags] of [
['build', ['[--stdin]', '[-o <file.plan>]', '[--json]', '[--errors-only]', '[--map (requires -o)]', '[--entry <kind> (requires -o; no --map or --stdin)]', '[--surface terminal (no --entry)]']],
['verify', ['[--runner-only]', '[--js]', '[--walks N]', '[--steps M]', '[--walk-seed N]', '[--confirm N]', '[--types]', '[--prove <Module>]']],
]) {
const line = text.split('\n').find(line => line.startsWith(` ${command} <file.contract>`));
assert.ok(line, `contract help names ${command}`);
for (const flag of flags) assert.ok(line.includes(flag), `${command} help omits ${flag}: ${line}`);
}
});
// LLP 1116.003 D5: the helps a small web app reads first (web, test, agent) say what a first build
// needs; `--help all` names every flag the short text leaves out, and the ports.
test('a first build\'s helps are short, and --help all has every flag', () => {
for (const verb of ['web', 'test', 'agent']) {
const short = helpText(verb), all = helpText(verb, { all: true });
assert.ok(short.length <= 1800, `${verb} --help is ${short.length} bytes`);
assert.match(short, new RegExp(`^More bun exact\\.mjs ${verb} --help all: `, 'm'));
for (const [row] of [...HELP[verb].flags, ...HELP[verb].more]) {
for (const spec of row.split(' · ')) assert.ok(all.includes(spec.split(' ')[0]), `${verb} --help all names ${spec}`);
}
assert.match(all, /^Ports {4}/m, `${verb} --help all says its ports`);
assert.doesNotMatch(all, /^More /m);
}
// LLP 1116.008 D2: web's help does not teach a background dev server (builders followed it into five steps).
assert.match(helpText('web', { all: true }), /The dev loop is for a person watching\n\s+the app change: verify, test and agent build and serve their own copy, so checking\n\s+the app needs no dev server\./);
assert.doesNotMatch(helpText('web', { all: true }), /web\.log| &\s|background|\$!/);
// The server restarts per test, including when it keeps its state only in memory.
assert.match(helpText('test'), /^Server +restarts empty per test .*database:false too/m);
assert.match(helpText('test'), /^Server +.*ancestor.*shared/m);
assert.match(helpText('verify', { all: true }), /Server +.*restarts empty per test/);
assert.match(helpText('verify', { all: true }), /relaunch replays the chosen test/);
assert.match(helpText('verify', { all: true }), /ancestor.*shared/);
// LLP 1116.007 D2: a builder parses `agent` output, so the short text says --json is one object per line.
assert.match(helpText('agent'), /^ {2}--json +one compact JSON object per op reply, one per line/m);
// LLP 1116.007 D3.9: `state <name>` is a state or resource read down a dotted path; a big one prints its sizes.
assert.match(helpText('agent'), /^ {2}state \[<name>\] \[--full\] +a state, resource or field, dotted: groceries\.0\.done; >8 KB: sizes/m);
});
// LLP 1116.009 D6: builders read `help` and then four or five verbs' `--help`, each a step carried to the end of the build.
// The index is the one read: each verb, then the forms a build uses, in one screen, and every flag a form names is one its
// verb's script parses (a form never names a flag that does not exist).
test('help is the one read: the verbs and the forms a build uses, in one screen, naming only flags the verbs parse', () => {
for (const verb of Object.keys(HELP)) for (const all of [false, true]) assert.doesNotMatch(helpText(verb, { all }), /undefined/, verb);
// The index an app's exact.mjs prints, and a game's.
const [text, gameText] = [false, true].map((game) => indexText({ verbs: Object.keys(appVerbs('app', '.', { game })) }));
for (const index of [text, gameText]) {
assert.ok(index.split('\n').length <= 36, `the index is ${index.split('\n').length} lines`);
assert.ok(Buffer.byteLength(index) <= 2800, `the index is ${Buffer.byteLength(index)} bytes`);
}
// What builders used across rounds 10 and 11: verify's platform and `--shots`, test's platform, agent's `--storage`, `--size`
// and its common operations, web and ios `--run`, contract's build and vocab, pattern.
for (const form of [
/^ {2}verify \[ios\] \[--shots <dir>\]/m, /^ {2}test \[web\|ios\]/m, /^ {2}agent \[web\|ios\] \[--storage <name>\] \[--size <w>x<h>\]/m,
/"tap <target>"/, /"type <target> <text>"/, /"clock data\|settle\|\+<ms>"/, /"state \[<name>\]"/, /"screenshot <png>"/, /"check \[both\]"/, /"resize <w>x<h>"/, /"prefer <feature> <value>"/,
/^ {2}web \[--port <n>\]/m, /^ {2}ios \[--run\]/m, /^ {2}contract build <file> --json · contract vocab <name>/m, /^ {2}pattern \[<name>\]/m,
]) assert.match(text, form);
for (const [verb, entry] of Object.entries(HELP)) {
const known = knownFlags(verb);
for (const form of entry.common ?? []) {
for (const [flag] of form.matchAll(/--[a-z][a-z-]*/g)) {
if (verb === 'contract') continue; // the compiler's own subcommand flags
assert.ok(known.has(flag) || [...HELP[verb].flags, ...(HELP[verb].more ?? [])].some(([row]) => row.includes(flag)), `${verb}'s form names ${flag}, which its script does not parse`);
}
}
}
// Without a verb's forms (a game's own verbs have none), the index is the verbs.
assert.doesNotMatch(indexText({ verbs: ['lock'] }), /The forms a build uses/);
});
test('unsigned device builds and archives refuse incompatible options before building', async () => {
const { appleBuildRefusal } = await import('./help.mjs');
for (const signed of [false, true]) for (let mask = 0; mask < 32; mask++) {
const flags = ['--unsigned', '--archive', '--run', '--device', '--host'];
const args = flags.filter((_, bit) => mask & (1 << bit));
const [unsigned, archive, run, device, host] = flags.map(flag => args.includes(flag));
const env = signed ? { EXACT_PROFILE: 'profile', EXACT_IDENTITY: 'identity' } : {};
const refusal = appleBuildRefusal(args, env, 'darwin');
assert.equal(refusal == null, !(unsigned && (!device || run) || archive && (!device || run || host || !unsigned && !signed)), JSON.stringify({ args, signed, refusal }));
}
assert.match(appleBuildRefusal(['--device', '--unsigned', '--run'], {}, 'darwin'), /ad-hoc signed device bundle cannot launch on a stock device/);
for (const args of [['--watchos', '--unsigned'], ['--watchos', '--device', '--unsigned']]) assert.match(appleBuildRefusal(args, {}, 'darwin'), /--unsigned takes no --watchos/);
for (const args of [['--device', '--development'], ['--device', '--unsigned', '--development'], ['--device', '--archive', 'a.ipa', '--development']]) assert.match(appleBuildRefusal(args, {}, 'darwin'), /--development needs --archive and --unsigned/);
const development = ['--device', '--archive', 'a.ipa', '--unsigned', '--development'];
assert.equal(appleBuildRefusal(development, {}, 'darwin'), null);
for (const [args, env] of [[[...development, '--release'], {}], [development, { EXACT_UPDATE_TRUST: 'production' }]]) assert.match(appleBuildRefusal(args, env, 'darwin'), /takes neither --release/);
assert.match(appleBuildRefusal(['--ios', '--run'], {}, 'linux'), /Linux has neither/);
assert.match(appleBuildRefusal(['--device', '--unsigned'], {}, 'linux'), /--device on Linux needs --archive/);
assert.equal(appleBuildRefusal(['--device', '--archive', 'a.ipa', '--unsigned'], {}, 'linux'), null);
assert.equal(appleBuildRefusal(['--ios'], {}, 'linux'), null);
const result = spawnSync(process.execPath, ['host/apple/build.mjs', '--device', '--unsigned', '--run', 'no-such-app'], { encoding: 'utf8' });
assert.equal(result.status, 1);
assert.match(result.stderr, process.platform === 'linux' ? /--run needs an iOS simulator/ : /--unsigned takes no --run/);
assert.doesNotMatch(result.stderr, /no app at/);
});