Compare commits

..
49 changed files with 1565 additions and 324 deletions

No files matched your search

-25
View File
@@ -6,31 +6,6 @@ import {
} from "../config/runtimeAnalysisSettings.js";
import { parseConfig } from "../config/parseConfig.js";
it.each([
["inspect_native_api", [64, 7]],
["trace_native_ui_action", []],
] as const)(
"reports the decompiler budgets consumed by %s",
(operation, values) => {
const store = new AnalysisSettings({
REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "7",
REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB: "64",
});
expect(
store
.inspect({
provider: "ghidra",
operation,
settings: [
"ghidra.decompile_max_payload_mib",
"ghidra.decompile_timeout_seconds",
],
})
.settings.map(({ value }) => value),
).toEqual(values);
},
);
it("updates atomically, restores launch values, and refuses stale or invalid writes", () => {
const store = new AnalysisSettings({ REA_ASAR_MAX_HEADER_BYTES: "20000000" });
const inspect = () => store.inspect({ settings: ["asar.max_header_bytes"] });
@@ -25,6 +25,7 @@ import {
import { pageBound } from "../../domain/analysisView/environmentStatePaging.js";
import { replayReason } from "../../domain/analysisView/environmentStateVariants.js";
import { isAbsence } from "../../domain/analysisView/environmentStateWriteChecks.js";
import { updatedEntity } from "../../domain/analysisView/environmentStateWriteApply.js";
import {
evaluateQuery,
renderQueryList,
@@ -291,16 +292,6 @@ export const softDeleteFlag = (
return undefined;
};
/** Fields as an application that soft-deletes shows them once deleted: with the type's deleted flag set, when it has one. */
const deletedFields = (
serving: StateServing,
type: string,
fields: JsonObject,
): JsonObject => {
const flag = softDeleteFlag(serving, type);
return flag === undefined ? fields : { ...fields, [flag]: true };
};
/** Fields as a delete's answer shows them: with the type's deleted flag as its recorded answer `shown` gave it, if it did. */
export const deleteAnswerFields = (
serving: StateServing,
@@ -326,9 +317,11 @@ const deletedEntity = (
call,
(collection, id) => store.deleted(type, collection, id),
);
return entity === undefined
? undefined
: { ...entity, fields: deletedFields(serving, type, entity.fields) };
if (entity === undefined) return undefined;
const flag = softDeleteFlag(serving, type);
return flag === undefined
? entity
: updatedEntity(entity, { ...entity.fields, [flag]: true });
};
/** The answer for an entity the state does not hold: the tool's latest recorded 400, 404 or 410, else a tool error. */
@@ -33,6 +33,44 @@ const objectOf = (body: JsonValue | undefined): Record<string, JsonValue> => {
return body;
};
describe("served path-derived fields", () => {
it.each([false, true])(
"preserves repeated parameter occurrences (ambiguous: %s)",
(ambiguous) => {
const project = ambiguous ? 1 : 3;
const path = `/api/orgs/1/projects/${project}/tasks`;
const first = { id: 1, name: "Old", orgId: 1, projectId: project };
const made = { id: 2, name: "Recorded", orgId: 1, projectId: project };
const tasks = servedFrom([
["GET", path, [first]],
["GET", `${path}/1`, first],
["GET", "/api/orgs/2/projects/4/tasks", []],
["POST", path, made, 201, { name: "Recorded" }],
["GET", `${path}/2`, made],
["GET", path, [first, made]],
]);
const tool = "post_orgs_projects_tasks_by_id_id_2";
const args = { id: "2", id_2: "4", body: { name: "Elsewhere" } };
if (ambiguous) {
// Equal recorded values establish no choice between the path positions.
const before = tasks.snapshot();
expect(tasks.call(tool, args).ok).toBe(false);
expect(tasks.snapshot()).toBe(before);
expect(
tasks.answer(tool, { id: "1", id_2: "1", body: { name: "Here" } })
.body,
).toMatchObject({ name: "Here", orgId: 1, projectId: 1 });
} else {
expect(tasks.answer(tool, args).body).toMatchObject({
name: "Elsewhere",
orgId: 2,
projectId: 4,
});
}
},
);
});
describe("served values an application derives from others", () => {
it("stamps a created tag's clocks with trailing fraction zeros dropped, as the capture spelled them", () => {
const tags = servedFrom(TAG_ROWS);
@@ -181,3 +181,47 @@ describe("served writes modelled on later reads", () => {
});
});
});
it.each([
["item", SOFT_DELETED_ROWS, "delete_contacts_by_id", { id: "1" }],
["bulk", STALE_BULK_DELETE_ROWS, "delete_contacts", { body: [{ id: 1 }] }],
] as const)(
"invalidates the deletion flag override after a served %s delete",
(_kind, rows, tool, args) => {
const contacts = servedFrom(rows);
const snapshot = contacts.store.snapshot();
expect(
contacts.store.replace({
rea_state: {
...snapshot.rea_state,
entities: snapshot.rea_state.entities.map((entity) =>
entity.id === "1"
? {
...entity,
view_fields: [
{
tool: "get_contacts_by_id",
fields: { deleted: false, name: "Item display" },
},
],
}
: entity,
),
},
}).ok,
).toBe(true);
expect(
contacts.answer("get_contacts_by_id", { id: "1" }).body,
).toMatchObject({
deleted: false,
});
contacts.answer(tool, args);
expect(
contacts.answer("get_contacts_by_id", { id: "1" }).body,
).toMatchObject({
id: 1,
name: "Item display",
deleted: true,
});
},
);
@@ -32,6 +32,7 @@ import {
} from "../../domain/analysisView/environmentStateWriteLinks.js";
import {
pathValuesOf,
pathParameterKeys,
type GeneratedRule,
type StateWritePlan,
type WriteCall,
@@ -106,14 +107,12 @@ export const pathValues = (
args: ReplayArguments,
): Map<string, string> => {
const names = environmentToolArguments(tool).path;
const template = tool.path_template
.slice(tool.path_template.indexOf("/") + 1)
.split("/");
const keys = pathParameterKeys(tool.path_template);
const values = new Map<string, string>();
for (const param of tool.params) {
const value = ownMember(args, names.get(param.name) ?? param.name);
const segment = template[param.index] ?? "";
if (typeof value === "string") values.set(segment.slice(1, -1), value);
const key = keys.get(param.index);
if (typeof value === "string" && key !== undefined) values.set(key, value);
}
return values;
};
@@ -11,6 +11,8 @@ import {
type Book,
} from "../../../tests/fixtures/environmentStateWrites.js";
import { servedCatalog } from "./environmentState.js";
import { environmentToolAnswer } from "./environmentToolAnswer.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
const RENAMED = renamedBook(FIRST_BOOK, "Renamed Book", 3);
@@ -72,6 +74,33 @@ const toolOf = (rows: readonly TimedCaptureRow[], name: string) => {
};
describe("served write queries", () => {
it.each([
"?notify=quiet&unmodeled=yes",
"?notify=quiet&notify=quiet",
"?notify=quiet&unmodeled=yes&unmodeled=no",
])("refuses unsupported observed query %s before changing state", (query) => {
const { served } = toolOf(PINNED_ROWS, "patch_books_by_id");
const tool = served.source.manifest.tools.find(
(candidate) => candidate.name === "patch_books_by_id",
);
if (tool === undefined) throw new Error("Expected patch tool");
const before = served.snapshot();
const pending = served.store.pendingWrite();
const respond = environmentToolAnswer(served.source, served.store);
const result = respond(
tool,
{ id: "2", notify: "quiet", body: { name: "Changed" } },
query,
);
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected pinned-query refusal");
expect(projectAnalysisError(result.error).message).toMatch(
/query|repeated/u,
);
expect(served.snapshot()).toBe(before);
expect(served.store.pendingWrite()).toEqual(pending);
});
it("advertises the recorded query as optional so omission reaches its default", () => {
const { served, tool } = toolOf(PINNED_ROWS, "patch_books_by_id");
expect(tool.inputSchema.properties["notify"]).toMatchObject({
@@ -1,5 +1,6 @@
import { pinnedQuery } from "../../domain/analysisView/environmentStateWriteStamps.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import { iterateUrlQueryParameters } from "../../domain/recordedTransportUrl.js";
import { err, ok, type Result } from "../../domain/result.js";
import { refused, type StateServing } from "./environmentStateAnswer.js";
import { callQuery } from "./environmentStateCallLines.js";
@@ -15,7 +16,16 @@ export const pinnedRequest = (
request: WriteRequest,
): Result<WriteRequest, AnalysisError> => {
const { tool, plan, args } = request;
if (hasRepeatedQueryArgument(tool, args))
const supplied = callQuery(tool, args);
const observed = new Map<string, string>();
let repeated = hasRepeatedQueryArgument(tool, args);
for (const [name, value] of iterateUrlQueryParameters(
request.observedQuery ?? "",
)) {
if (observed.has(name)) repeated = true;
observed.set(name, value);
}
if (repeated)
return err(
refused(
tool,
@@ -24,7 +34,9 @@ export const pinnedRequest = (
);
const query = pinnedQuery(
serving.source.exchanges.get(plan.exemplar_ordinal ?? -1)?.query,
callQuery(tool, args),
// Comparison supplies the raw query as well as translated known arguments.
// Keep unmodeled parameters visible without replacing those translations.
new Map([...observed, ...supplied]),
);
if (!query.ok) return err(refused(tool, `Tool ${tool.name} ${query.error}`));
const names = environmentToolArguments(tool).query;
+14 -12
View File
@@ -57,16 +57,6 @@ const timer = {
unit: "ms",
appliesAt: "next_operation",
} as const;
/** Operations that invoke Ghidra's decompiler directly or through a workflow. */
const ghidraDecompileOperations = [
"procedure_pseudo_code",
"analyze_function",
"inspect_native_api",
"decompile_all",
"trace_native_values",
] as const;
/** Metadata aggregates existing owner definitions; no environment mutation or engine discovery. */
export const ANALYSIS_SETTINGS = {
"ghidra.decompile_max_payload_mib": {
@@ -77,7 +67,14 @@ export const ANALYSIS_SETTINGS = {
unit: "MiB",
appliesAt: "next_operation",
provider: "ghidra",
operations: ghidraDecompileOperations,
operations: [
"procedure_pseudo_code",
"analyze_function",
"inspect_native_api",
"decompile_all",
"trace_native_values",
"trace_native_ui_action",
],
},
"asar.max_listing_bytes": {
environment: "REA_ASAR_MAX_LISTING_BYTES",
@@ -215,7 +212,12 @@ export const ANALYSIS_SETTINGS = {
unit: "seconds",
appliesAt: "next_operation",
provider: "ghidra",
operations: ghidraDecompileOperations,
operations: [
"procedure_pseudo_code",
"analyze_function",
"decompile_all",
"trace_native_values",
],
},
"jadx.operation_timeout_ms": {
...timer,
@@ -120,3 +120,48 @@ it("charges the group key before building it so an oversized request method fail
});
expect(finished(accumulator).tools).toHaveLength(0);
});
it("counts only admitted scenario transactions and refreshes their snapshot limitations", () => {
const accumulator = new EnvironmentManifestAccumulator(32 * 1024);
expect(finished(accumulator).origin).toBe("");
expect(accumulator.add(sample(0, "/api/items"), "absent").ok).toBe(true);
const before = finished(accumulator);
expect(before.limitations.join(" ")).not.toContain("no primary origin");
expect(
accumulator.add(
{
...sample(1, "/api/items", `{"name":"${"x".repeat(2_000)}"}`),
limitations: ["rejected-only limitation"],
},
"redacted",
).ok,
).toBe(false);
expect(finished(accumulator)).toEqual(before);
expect(
accumulator.add({
...sample(1, "/api/items"),
limitations: ["uncommitted limitation", "x".repeat(100_000)],
}).ok,
).toBe(false);
expect(finished(accumulator)).toEqual(before);
expect(
accumulator.add(
{
...sample(2, "/api/items", '{"name":"n"}'),
limitations: ["accepted record limitation"],
},
"redacted",
).ok,
).toBe(true);
const after = finished(accumulator);
expect(after.limitations).toContain("accepted record limitation");
expect(after.records.map((record) => record.ordinal)).toEqual([0, 2]);
expect(
after.limitations.filter((limitation) =>
limitation.includes("scenario transactions with a usable request line"),
),
).toEqual([
"1 of 2 scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.",
"1 of 2 scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.",
]);
});
@@ -215,9 +215,11 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
record: WebNetworkCaptureRecord,
scenarioResponseBody?: ScenarioResponseBodyRetention,
): Result<void, AnalysisError> {
const limitations = this.addLimitations(record.limitations);
if (!limitations.ok) return limitations;
const rollbackAdmission = this.retainedCheckpoint();
let committed = false;
try {
const limitations = this.prepareLimitations(record.limitations);
if (!limitations.ok) return limitations;
const line = readRecordedRequestLine(
record.reported,
this.includeQueryParameterNames,
@@ -226,16 +228,14 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
if (line === null) return this.overBudget();
const releaseGrouping = this.temporaryCheckpoint();
if (line.kind === "excluded") {
this.#unrepresented += 1;
releaseGrouping();
return this.onRecordedRequestLine(record, line);
}
// Coverage uses the same admitted request line as grouping, including origin and raw path.
if (scenarioResponseBody !== undefined) {
this.#scenarioTransactions += 1;
if (scenarioResponseBody === "absent") this.#scenarioWithoutBody += 1;
if (scenarioResponseBody === "redacted")
this.#scenarioRedactedBodies += 1;
const prepared = this.prepareRecordedRequestLine(record, line);
if (!prepared.ok) return prepared;
prepared.value();
limitations.value();
this.#unrepresented += 1;
committed = true;
return ok(undefined);
}
const route = scanRouteTemplate(line.path, this.scratch);
if (route === null) return this.overBudget();
@@ -266,7 +266,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
? this.#bodyShape(record.reported)
: ok(null);
if (!bodyShape.ok) return bodyShape;
const grouped = this.#append(
const grouped = this.#prepareAppend(
record.ordinal,
line,
route,
@@ -277,18 +277,33 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
if (!grouped.ok) return grouped;
// Keep the parsed line's admission while releasing grouping-only scratch.
releaseGrouping();
return this.onRecordedRequestLine(record, line);
const prepared = this.prepareRecordedRequestLine(record, line);
if (!prepared.ok) return prepared;
// Both owners admitted their facts; neither commit can reject the record.
grouped.value();
prepared.value();
limitations.value();
// Coverage uses the same admitted request line as grouping, including origin and raw path.
if (scenarioResponseBody !== undefined) {
this.#scenarioTransactions += 1;
if (scenarioResponseBody === "absent") this.#scenarioWithoutBody += 1;
if (scenarioResponseBody === "redacted")
this.#scenarioRedactedBodies += 1;
}
committed = true;
return ok(undefined);
} finally {
if (!committed) rollbackAdmission();
this.releaseTemporary();
}
}
/** Consume the owner's parsed request line after successful grouping, including exclusions. */
protected onRecordedRequestLine(
/** Admit derived facts from the owner's parsed line, then commit only after all owners admit it. */
protected prepareRecordedRequestLine(
_record: WebNetworkCaptureRecord,
_line: RecordedRequestLine,
): Result<void, AnalysisError> {
return ok(undefined);
): Result<() => void, AnalysisError> {
return ok(() => {});
}
/** Read the top-level key types of the group's first recorded JSON-typed request body; nested values are never walked. */
@@ -316,14 +331,14 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
return ok({ type: "object", properties: Object.fromEntries(entries) });
}
#append(
#prepareAppend(
ordinal: number,
line: RequestLine,
route: RouteTemplate,
queryNames: ReadonlySet<string> | undefined,
bucket: { readonly key: string; readonly group: GroupState | undefined },
exchange: { readonly shape: BodyShape | null; readonly empty: boolean },
): Result<void, AnalysisError> {
): Result<() => void, AnalysisError> {
const bodyShape = exchange.shape;
const mime = mediaTypeEssence(line.rawMime);
let group = bucket.group;
@@ -366,48 +381,49 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
bytes += ENTRY_BYTES + textBytes(name.length);
// Reserve the whole mutation while route scratch is still live.
if (!this.retain(bytes)) return this.overBudget();
const position = this.#records.length;
this.#records.push({
ordinal,
method: line.method,
path: line.path,
status: line.status,
contentType: mime,
});
const jsonLike = isJsonLike(mime);
const stats = origin ?? { records: 0, jsonLike: 0 };
stats.records += 1;
if (jsonLike) stats.jsonLike += 1;
this.#origins.set(line.origin, stats);
if (group === undefined) {
group = {
return ok(() => {
const position = this.#records.length;
this.#records.push({
ordinal,
method: line.method,
origin: line.origin,
api_surface_template: route.template,
parts: routeParts(route),
positions: [],
statuses: new Set(),
contentTypes: new Set(),
queryNames: new Set(),
bodyShape,
apiPath: isApiPath(route.template),
jsonLike: false,
bodiless: BODILESS_WRITE_METHODS.has(line.method),
};
this.#groups.set(bucket.key, group);
} else {
for (const segment of route.segments) {
const part = group.parts[segment.segment_index];
if (part?.kind === "param") part.values.add(segment.value);
path: line.path,
status: line.status,
contentType: mime,
});
const jsonLike = isJsonLike(mime);
const stats = origin ?? { records: 0, jsonLike: 0 };
stats.records += 1;
if (jsonLike) stats.jsonLike += 1;
this.#origins.set(line.origin, stats);
if (group === undefined) {
group = {
method: line.method,
origin: line.origin,
api_surface_template: route.template,
parts: routeParts(route),
positions: [],
statuses: new Set(),
contentTypes: new Set(),
queryNames: new Set(),
bodyShape,
apiPath: isApiPath(route.template),
jsonLike: false,
bodiless: BODILESS_WRITE_METHODS.has(line.method),
};
this.#groups.set(bucket.key, group);
} else {
for (const segment of route.segments) {
const part = group.parts[segment.segment_index];
if (part?.kind === "param") part.values.add(segment.value);
}
}
}
group.positions.push(position);
group.jsonLike ||= jsonLike;
group.bodiless &&= exchange.empty;
if (line.status !== null) group.statuses.add(line.status);
if (mime !== null) group.contentTypes.add(mime);
for (const name of queryNames ?? []) group.queryNames.add(name);
return ok(undefined);
group.positions.push(position);
group.jsonLike ||= jsonLike;
group.bodiless &&= exchange.empty;
if (line.status !== null) group.statuses.add(line.status);
if (mime !== null) group.contentTypes.add(mime);
for (const name of queryNames ?? []) group.queryNames.add(name);
});
}
#tools(groups: readonly ToolGroup[]): Tool[] {
@@ -513,7 +529,8 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
compare(left.method, right.method) ||
compare(left.reason, right.reason),
);
const limitations = this.addLimitations([
// Derived limitations describe this snapshot, not an earlier finish.
const limitations = this.snapshotLimitations([
...(this.reportsBodyShapes &&
tools.some((tool) => tool.body_shape !== null)
? [
@@ -521,16 +538,6 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
]
: []),
...this.#segmentLimitLimitations(toolGroups),
...(this.#scenarioWithoutBody > 0
? [
`${String(this.#scenarioWithoutBody)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.`,
]
: []),
...(this.#scenarioRedactedBodies > 0
? [
`${String(this.#scenarioRedactedBodies)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.`,
]
: []),
...(this.#unrepresented > 0
? [
`${String(this.#unrepresented)} retained records without a usable request line are not represented in the manifest.`,
@@ -541,6 +548,16 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
"No retained record had a parseable origin; the manifest names no primary origin.",
]
: []),
...(this.#scenarioWithoutBody > 0
? [
`${String(this.#scenarioWithoutBody)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.`,
]
: []),
...(this.#scenarioRedactedBodies > 0
? [
`${String(this.#scenarioRedactedBodies)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.`,
]
: []),
]);
if (!limitations.ok) return limitations;
return ok({
@@ -567,7 +584,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
tools.filter((tool) => tool.params.length > 0).length +
excluded.filter((exclusion) => exclusion.templated).length,
coverage: completeWithinViewCoverage(tools.length, tools.length),
limitations: this.sortedLimitations(),
limitations: limitations.value,
});
}
}
@@ -3,6 +3,7 @@ import type { AnalysisError } from "../analysisErrorBase.js";
import { parseExactJson } from "../exactJsonNumber.js";
import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js";
import {
addOffsetDateTimeMilliseconds,
compareOffsetDateTimeInstants,
readRecordedStart,
} from "../offsetDateTime.js";
@@ -29,6 +30,7 @@ import {
base64PrefixChars,
readRecordedResponseBody,
readRecordedResponseDeclaration,
recordedResponseStatus,
recordedResponseBodyIsEmpty,
type RecordedResponseBody,
} from "./recordedResponseBody.js";
@@ -45,6 +47,9 @@ import {
* room under the JSON depth limit to be sealed and served.
*/
const STATE_NESTING_DEPTH = 16;
// Number.MIN_VALUE milliseconds needs this many decimal places in seconds;
// this also covers the whole-second digits of every finite number duration.
const DURATION_DIGITS = 3 - Math.floor(Math.log10(Number.MIN_VALUE));
/** A recorded response body as state JSON: a JSON-like media type, complete UTF-8 text within the state body limit, exact numbers; otherwise undefined. */
export const stateBodyJson = (
@@ -69,14 +74,14 @@ export const stateBodyJson = (
const startSchema = z.object({ startedDateTime: z.string() });
const durationSchema = z.object({ time: z.number().nonnegative() });
const redactedWriteSchema = z.object({
const unlocatedWriteSchema = z.object({
request: z.object({ method: z.string() }),
response: z.object({ status: z.number() }).optional(),
});
interface RecordEntry {
readonly facts: StateRecordFacts;
/** Null for a write whose URL was redacted: its target is unknown, so it bounds every type. */
/** Null for a write without a usable URL: its unknown target bounds every type. */
readonly origin: string | null;
readonly targetExclusion?: "url-redacted" | "url-unparseable";
readonly contentType: string;
readonly startInterpretation:
| NonNullable<ReturnType<typeof readRecordedStart>>["interpretation"]
@@ -119,10 +124,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
return false;
}
protected override onRecordedRequestLine(
protected override prepareRecordedRequestLine(
record: WebNetworkCaptureRecord,
line: RecordedRequestLine,
): Result<void, AnalysisError> {
): Result<() => void, AnalysisError> {
const started = startSchema.safeParse(record.reported);
const duration = durationSchema.safeParse(record.reported);
// Admit normalization and fraction parsing before materializing their strings.
@@ -148,28 +153,52 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
startMs === undefined || !duration.success
? undefined
: startMs + duration.data.time;
// Exact completion keeps arbitrary source precision. Reserve digit-array and
// string expansion before the shared decimal addition allocates them.
if (
startInstant !== undefined &&
duration.success &&
duration.data.time !== 0 &&
!this.scratch(
RECORD_SCRATCH_BYTES +
textBytes(Math.max(startInstant.fraction.length, DURATION_DIGITS)) *
3,
)
)
return this.overBudget();
const endInstant =
startInstant === undefined || !duration.success
? undefined
: addOffsetDateTimeMilliseconds(startInstant, duration.data.time);
const timing: StateRecordFacts["timing"] = {
start:
startInstant === undefined || startMs === undefined
? undefined
: { instant: startInstant, ms: startMs },
end: endInstant,
endMs: end !== undefined && Number.isFinite(end) ? end : undefined,
};
// A fraction substring may retain its normalized timestamp's backing store,
// including trailing zeroes removed from the exact comparison value.
const instantBytes =
started.success && startInstant !== undefined
(started.success && startInstant !== undefined
? ENTRY_BYTES + textBytes(started.data.startedDateTime.length + 6)
: 0;
: 0) +
(endInstant !== undefined && endInstant !== startInstant
? ENTRY_BYTES + textBytes(endInstant.fraction.length)
: 0);
if (line.kind === "excluded")
return line.reason === "url-redacted"
? this.#redactedWrite(
return line.reason === "url-redacted" ||
line.reason === "url-unparseable" ||
line.reason === "request-unavailable"
? this.#unlocatedWrite(
record,
line.reason === "url-redacted" ? "url-redacted" : "url-unparseable",
timing,
instantBytes,
reading?.interpretation,
)
: ok(undefined);
: ok(() => undefined);
// The entry keeps the line's strings and a reference to the already admitted record.
if (
!this.retain(
@@ -185,56 +214,62 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
)
)
return this.overBudget();
this.#entries.set(record.ordinal, {
facts: {
ordinal: record.ordinal,
method: line.method,
path: line.path,
query: line.query === "" ? undefined : line.query,
status: line.status,
timing,
},
origin: line.origin,
contentType: line.rawMime,
startInterpretation: reading?.interpretation,
record,
return ok(() => {
this.#entries.set(record.ordinal, {
facts: {
ordinal: record.ordinal,
method: line.method,
path: line.path,
query: line.query === "" ? undefined : line.query,
status: line.status,
timing,
},
origin: line.origin,
contentType: line.rawMime,
startInterpretation: reading?.interpretation,
record,
});
});
return ok(undefined);
}
/** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */
#redactedWrite(
/** Prepare a successful write with unknown target, whose empty root relates to every type. */
#unlocatedWrite(
record: WebNetworkCaptureRecord,
reason: "url-redacted" | "url-unparseable",
timing: StateRecordFacts["timing"],
instantBytes: number,
startInterpretation: RecordEntry["startInterpretation"],
): Result<void, AnalysisError> {
const parsed = redactedWriteSchema.safeParse(record.reported);
if (!parsed.success) return ok(undefined);
const status = parsed.data.response?.status ?? 0;
): Result<() => void, AnalysisError> {
const parsed = unlocatedWriteSchema.safeParse(record.reported);
if (!parsed.success) return ok(() => undefined);
const status = recordedResponseStatus(
readRecordedResponseDeclaration(record.reported),
);
const facts: StateRecordFacts = {
ordinal: record.ordinal,
method: parsed.data.request.method,
path: "",
query: undefined,
status: status > 0 ? status : null,
status,
timing,
};
if (!isStateWrite(facts)) return ok(undefined);
if (!isStateWrite(facts)) return ok(() => undefined);
if (
!this.retain(
ENTRY_BYTES * 8 + textBytes(facts.method.length) + instantBytes,
)
)
return this.overBudget();
this.#entries.set(record.ordinal, {
facts,
origin: null,
contentType: "",
startInterpretation,
record,
return ok(() => {
this.#entries.set(record.ordinal, {
facts,
origin: null,
targetExclusion: reason,
contentType: "",
startInterpretation,
record,
});
});
return ok(undefined);
}
/** Parse one retained response body as state JSON, charging its decoding and tree as temporary work. */
@@ -330,7 +365,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
entry.origin === manifest.value.origin || entry.origin === null,
);
const redactedWrites = entries.filter(
(entry) => entry.origin === null,
(entry) => entry.targetExclusion === "url-redacted",
).length;
const unparseableWrites = entries.filter(
(entry) => entry.targetExclusion === "url-unparseable",
).length;
const timeline: StateTimeline =
entries.length > 0 &&
@@ -361,7 +399,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
},
}).run();
if (state === null) return this.overBudget();
const limitations = this.addLimitations([
const limitations = this.snapshotLimitations([
...manifest.value.limitations,
...(entries.some((entry) => entry.startInterpretation === "native-date")
? [
"Some recorded start spellings use native Date parsing at millisecond precision; spellings without an explicit zone depend on the host timezone.",
@@ -372,6 +411,11 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
`${String(redactedWrites)} of the recorded writes had a redacted URL; each ends the initial state of every type, since its target is unknown.`,
]
: []),
...(unparseableWrites > 0
? [
`${String(unparseableWrites)} of the recorded writes had an unusable URL; each ends the initial state of every type, since its target is unknown.`,
]
: []),
...(timeline === "record-order"
? [
"Some records carry no start time, so the state's timeline is record order and overlapping requests are not detected.",
@@ -397,7 +441,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
...state,
unknowns: UNKNOWNS.map((unknown) => ({ ...unknown })),
coverage: manifest.value.coverage,
limitations: this.sortedLimitations(),
limitations: limitations.value,
});
}
}
@@ -1418,7 +1418,7 @@ describe("encoded parent spellings", () => {
path: "/api/orgs/uid/acme1/books",
query: undefined,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
};
const lines = checkedLines({
tools: [tool],
@@ -1621,7 +1621,7 @@ describe("resolved encoded item observations", () => {
path: "/api/orgs/acme1/books/%61",
query: undefined,
status: 204,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
};
const target = itemTarget(state, "/api/orgs/{org_id}/books", write);
const itemView = {
@@ -1642,7 +1642,7 @@ describe("resolved encoded item observations", () => {
path: "/api/orgs/acme1/books/%61",
query: undefined,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
},
];
const search: ObservationSearch = {
@@ -2308,23 +2308,53 @@ describe("reads and calls on an entity an unmodelled write left unknown", () =>
{},
];
it("keeps the entity unknown through a failed read that is no absence, for a later read to check", () => {
const rows: TimedCaptureRow[] = [
["GET", "/api/books", [named("A"), other]],
["GET", "/api/books/1", named("A")],
["GET", "/api/books/2", other],
["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }],
ARCHIVE,
// The application was briefly unavailable, so this answer shows nothing of the book.
["GET", "/api/books/1", { error: "unavailable" }, 503],
["GET", "/api/books/1", named("Z")],
];
expect(gated(rows, "patch_books_by_id")).toMatchObject({
served: true,
checked_record_ordinals: [6],
contradicted_by: null,
});
});
it.each([
{ label: "scalar", read: "unavailable", served: false },
{ label: "array", read: [], served: false },
{ label: "object", read: named("A"), served: true },
])(
"requires a usable item baseline after an unmodelled write ($label)",
({ read, served }) => {
const rows: TimedCaptureRow[] = [
["GET", "/api/books", [named("A"), other]],
["GET", "/api/books/1", named("A")],
["GET", "/api/books/2", other],
ARCHIVE,
["GET", "/api/books/1", read],
["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }],
["GET", "/api/books/1", named("Z")],
];
expect(gated(rows, "patch_books_by_id")).toMatchObject({
served,
checked_record_ordinals: served ? [6] : [],
});
},
);
it.each([
{ status: 503, body: { error: "unavailable" } },
{ status: 200, body: "unavailable" },
{ status: 200, body: [] },
])(
"keeps pending effects through an unusable item read ($status, $body)",
({ status, body }) => {
const rows: TimedCaptureRow[] = [
["GET", "/api/books", [named("A"), other]],
["GET", "/api/books/1", named("A")],
["GET", "/api/books/2", other],
["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }],
ARCHIVE,
// Neither failure nor a successful non-object response shows the book.
["GET", "/api/books/1", body, status],
["GET", "/api/books/1", named("Z")],
];
expect(gated(rows, "patch_books_by_id")).toMatchObject({
served: true,
checked_record_ordinals: [6],
contradicted_by: null,
});
},
);
it("confirms no update of a volatile field by a read that shows the entity again, as no other read does", () => {
const seen = (id: number, count: number) => ({
@@ -2468,7 +2498,7 @@ describe("entities an unmodelled write made", () => {
path: "/api/books/3",
query: undefined,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
};
expect(
unseen.adopt(view as unknown as StateItemView, { record, index: 1 }),
@@ -2493,7 +2523,7 @@ describe("entities an unmodelled write made", () => {
path: "/api/books/3",
query: undefined,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
},
index,
});
@@ -3201,7 +3231,7 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => {
path: "/api/books/%63",
query: undefined,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
};
const target = itemTarget(state, view.type, record);
expect(target.id).toBe("%63");
@@ -555,8 +555,6 @@ export class EnvironmentStateWriteSimulation {
): void {
const target = entityTarget(entity.type, entity.collection, entity.id);
const writes = this.#touched.get(target) ?? new Map();
this.#taint.known(entity.type, entity.collection, entity.id);
this.#touched.delete(target);
if (
isAbsence(record.status) ||
this.#input.io.withBody(record.ordinal, (body) =>
@@ -567,12 +565,16 @@ export class EnvironmentStateWriteSimulation {
),
)
) {
this.#taint.known(entity.type, entity.collection, entity.id);
this.#touched.delete(target);
this.#removeEntity(entity);
return;
}
const object = this.#checkFields(view, record, entity, writes);
if (object !== undefined)
this.#state.put(updatedEntity(entity, { ...entity.fields, ...object }));
if (object === undefined) return;
this.#taint.known(entity.type, entity.collection, entity.id);
this.#touched.delete(target);
this.#state.put(updatedEntity(entity, { ...entity.fields, ...object }));
}
/** Whether an item read shows an entity an unmodelled write made, or one left unknown, again rather than checks it. */
@@ -30,9 +30,9 @@ it("bounds query parser expansion before publishing or merging observed names",
const accumulator = new WebNetworkApiSurfaceAccumulator(64 * 1024, true);
add(accumulator, record(0, "/products?kept=1"));
const before = accumulator.page(0, 1);
expect(
accumulator.add(record(1, `/products?${"x=1&".repeat(200)}`)),
).toMatchObject({
const rejected = record(1, `/products?${"x=1&".repeat(200)}`);
rejected.limitations = ["Uncommitted rejected record caveat"];
expect(accumulator.add(rejected)).toMatchObject({
ok: false,
error: { _tag: "AnalysisResourceConstraintError" },
});
@@ -92,8 +92,10 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
}
add(record: WebNetworkCaptureRecord): Result<void, AnalysisError> {
const limitations = this.addLimitations(record.limitations);
const rollbackAdmission = this.retainedCheckpoint();
const limitations = this.prepareLimitations(record.limitations);
if (!limitations.ok) return limitations;
let committed = false;
try {
const line = readRecordedRequestLine(
record.reported,
@@ -101,14 +103,23 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
this.scratch,
);
if (line === null) return this.overBudget();
let appended: Result<void, AnalysisError>;
if (line.kind === "excluded")
return this.#exclude(record.ordinal, line.reason);
const route = scanRouteTemplate(line.path, this.scratch);
if (route === null) return this.overBudget();
const queryNames = readQueryParameterNames(line.query, this.scratch);
if (queryNames === null) return this.overBudget();
return this.#append(record.ordinal, line, route, queryNames);
appended = this.#exclude(record.ordinal, line.reason);
else {
const route = scanRouteTemplate(line.path, this.scratch);
if (route === null) return this.overBudget();
const queryNames = readQueryParameterNames(line.query, this.scratch);
if (queryNames === null) return this.overBudget();
appended = this.#append(record.ordinal, line, route, queryNames);
}
if (appended.ok) {
limitations.value();
committed = true;
}
return appended;
} finally {
if (!committed) rollbackAdmission();
this.releaseTemporary();
}
}
@@ -105,15 +105,54 @@ export abstract class WebNetworkProjectionAccumulator {
}
addLimitations(limitations: readonly string[]): Result<void, AnalysisError> {
for (const limitation of limitations) {
if (this.#limitations.has(limitation)) continue;
if (!this.retain(ENTRY_BYTES + textBytes(limitation.length)))
return this.overBudget();
this.#limitations.add(limitation);
}
const prepared = this.prepareLimitations(limitations);
if (!prepared.ok) return prepared;
prepared.value();
return ok(undefined);
}
/** Admit record qualifications without publishing them before the record commits. */
protected prepareLimitations(
limitations: readonly string[],
): Result<() => void, AnalysisError> {
const rollbackAdmission = this.retainedCheckpoint();
const pending = new Set<string>();
for (const limitation of limitations) {
if (this.#limitations.has(limitation) || pending.has(limitation))
continue;
if (!this.retain(ENTRY_BYTES + textBytes(limitation.length))) {
rollbackAdmission();
return this.overBudget();
}
pending.add(limitation);
}
return ok(() => {
for (const limitation of pending) this.#limitations.add(limitation);
});
}
/** Derive budgeted qualifications for this snapshot without retaining old snapshots. */
protected snapshotLimitations(
derived: readonly string[],
): Result<string[], AnalysisError> {
const release = this.temporaryCheckpoint();
if (
!this.scratch(
derived.reduce(
(bytes, limitation) =>
bytes + ENTRY_BYTES + textBytes(limitation.length),
0,
),
)
)
return this.overBudget();
try {
return ok([...new Set([...this.#limitations, ...derived])].sort(compare));
} finally {
release();
}
}
protected sortedLimitations(): string[] {
return [...this.#limitations].sort(compare);
}
@@ -67,6 +67,19 @@ const nativeParent = (
artifact: { path: "/fixtures/example.exe", sha256: "a".repeat(64) },
});
it("returns input errors for malformed parent Evidence identifiers before projection", () => {
for (const evidenceId of [`xx_${"a".repeat(64)}`, "ev_bad"])
expect(
projectAnalysisView(
{ ...nativeParent(), evidenceId },
{ kind: "summary" },
),
).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } });
expect(projectAnalysisView(nativeParent(), { kind: "summary" }).ok).toBe(
true,
);
});
it("projects native procedure and bounded assembly and high-pcode with exact parent identity", () => {
const parent = nativeParent();
const procedure = projectAnalysisView(parent, {
+3 -4
View File
@@ -53,7 +53,7 @@ import {
analysisViewInputError,
} from "./analysisViewErrors.js";
import { digestCanonicalValue } from "../canonicalDigest.js";
import { digestSchema } from "../digests.js";
import { prefixedDigestSchema } from "../digests.js";
import { err, ok, type Result } from "../result.js";
import type { AnalysisError } from "../analysisErrorBase.js";
import { safeParseOwnedBinaryLayout } from "../native/binaryLayout.js";
@@ -269,7 +269,7 @@ export const sealAnalysisView = (
view_digest: digestCanonicalValue(projection, "Analysis view"),
});
const parentDigest = (evidenceId: string): string => evidenceId.slice(3);
const parentEvidenceIdSchema = prefixedDigestSchema("ev");
const parseJavaScriptParent = (
input: unknown,
@@ -473,8 +473,7 @@ export const projectAnalysisView = (
view: AnalysisViewRequest,
accumulators: AnalysisViewAccumulators = {},
): Result<AnalysisViewResult, AnalysisError> => {
const digest = parentDigest(parent.evidenceId);
if (!digestSchema.safeParse(digest).success)
if (!parentEvidenceIdSchema.safeParse(parent.evidenceId).success)
return err(
analysisViewInputError([
{
@@ -3,6 +3,7 @@ import {
checkItemPlan,
type StateGateContext,
} from "./environmentStateGate.js";
import { locateItem, locateItems } from "./environmentStateRules.js";
import type {
JsonObject,
StateEntity,
@@ -21,7 +22,7 @@ const check = (
path: `/api/books/${ordinal + 1}`,
query: undefined,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
}));
const bodies: JsonObject[] = clocks.map(([, stamp], index) => ({
id: index + 1,
@@ -116,3 +117,37 @@ describe("clock spelling facts at the read gate", () => {
expect(large.bytes).toBe(small.bytes);
});
});
describe("item response identity selection", () => {
it("uses the known identity field before choosing an envelope tier", () => {
const body = { request_id: "42", data: { id: "42", name: "Book" } };
expect(locateItem(body, "42", "id")).toEqual({
pointer: "/data",
keys: ["id"],
});
expect(locateItem(body, "42")).toEqual({
pointer: "",
keys: ["request_id"],
});
expect(locateItem(body, "42", "uid")).toBeUndefined();
});
it("retains ambiguity between objects carrying the known identity", () => {
const body = { request_id: "42", left: { id: "42" }, right: { id: "42" } };
expect(locateItem(body, "42", "id")).toBeUndefined();
expect(locateItems(body, "42", "id")).toEqual([
{ pointer: "/left", keys: ["id"] },
{ pointer: "/right", keys: ["id"] },
]);
expect(locateItems(body, "42", "uid")).toEqual([]);
});
it("admits an explicitly known custom identity in a sole array element", () => {
const body = { data: [{ bookNumber: "42", name: "Book" }] };
expect(locateItem(body, "42", "bookNumber")).toEqual({
pointer: "/data/0",
keys: ["bookNumber"],
});
expect(locateItem(body, "42")).toBeUndefined();
});
});
@@ -108,3 +108,75 @@ it("takes no nested member for a counter unless it equals the number listed", ()
},
);
});
it.each([{ nextCursor: "cursor-2" }, { has_more: true }, { nextPage: 2 }])(
"keeps membership recorded when paging metadata says more items follow: %j",
(pageInfo) => {
const state = environmentStateViewOf(
timedCaptureEvidence([
["GET", "/api/records", { list: SEEDED, pageInfo }],
...SEEDED.map((item): TimedCaptureRow => [
"GET",
`/api/records/${String(item["Id"])}`,
item,
]),
]),
);
expect(
state.views.find((view) => view.tool === "get_records"),
).toMatchObject({
shape: "list",
membership: "recorded",
});
},
);
it("keeps completed paging envelopes whole and does not read entity fields as continuation metadata", () => {
const items = SEEDED.map((item): JsonObject => ({
...item,
nextCursor: "entity-value",
}));
const state = environmentStateViewOf(
timedCaptureEvidence([
[
"GET",
"/api/records",
{ list: items, pageInfo: { nextCursor: null, has_more: false } },
],
...items.map((item): TimedCaptureRow => [
"GET",
`/api/records/${String(item["Id"])}`,
item,
]),
]),
);
expect(state.views.find((view) => view.tool === "get_records")).toMatchObject(
{
shape: "list",
membership: "all",
},
);
});
it("keeps a direction-only recording on its recorded members instead of learning an unused direction", () => {
const state = environmentStateViewOf(
timedCaptureEvidence([
["GET", "/api/records", SEEDED],
["GET", "/api/records?direction=desc", SEEDED],
...SEEDED.map((item): TimedCaptureRow => [
"GET",
`/api/records/${String(item["Id"])}`,
item,
]),
]),
);
const view = state.views.find(
(candidate) => candidate.tool === "get_records",
);
expect(view).toMatchObject({
shape: "list",
membership: "recorded",
checked_record_ordinals: [0, 1],
});
expect(view).not.toHaveProperty("query.parameters.direction");
});
@@ -72,7 +72,7 @@ export const listFacts = (
counted.some(
([, value]) => recordedItemCount(value) !== items.length,
) ||
Object.keys(parent).some((name) => continues(name, parent[name]))
hasContinuation(parent, plan.pointer)
)
return;
const ids = items.map((item) => scalarText(ownMember(item, field)) ?? "");
@@ -150,3 +150,15 @@ const continues = (name: string, value: JsonValue | undefined): boolean =>
value !== null &&
value !== false &&
value !== "";
/** Continuation facts in the envelope or its sibling paging objects, never inside the list payload. */
const hasContinuation = (parent: JsonObject, listPointer: string): boolean => {
const root = pointerTokens(listPointer)[0];
return Object.entries(parent).some(
([name, value]) =>
continues(name, value) ||
(name !== root &&
isObject(value) &&
Object.entries(value).some(([inner, flag]) => continues(inner, flag))),
);
};
@@ -376,6 +376,16 @@ describe("queries the state can always answer", () => {
});
describe("parameters that ask for an order", () => {
it("cannot apply a direction without a sort field, even when the recorded order happens to match", () => {
expect(explainLine(line("?direction=desc", [1, 2, 3, 4]))).toBeUndefined();
const parameters = { direction: { role: "direction" } } as const;
expect(ids("?direction=asc", parameters)).toBeUndefined();
expect(ids("?direction=desc", parameters)).toBeUndefined();
expect(
ids("?sort=id&direction=desc", { ...parameters, sort: { role: "sort" } }),
).toEqual(["4", "3", "2", "1"]);
});
it("reads a sort field in any spelling as a sort, never as a parameter without effect", () => {
expect(explainLine(line("?_sort=id&_order=DESC", [4, 3, 2, 1]))).toEqual({
_sort: { role: "sort" },
@@ -206,7 +206,11 @@ const readCall = (
call.page = { index: number, first: parameter.first };
else return undefined;
}
return call;
// A direction cannot identify the field being ordered. Leaving it unused
// would turn a recorded direction into permission to answer its opposite.
return call.direction !== undefined && call.sort === undefined
? undefined
: call;
};
/** Entities in the order a sort value names, ties in state order, and those whose place state order decides: all without a sort, else those tied with a neighbour; undefined when it names no field. */
@@ -51,6 +51,12 @@ const readOnlyOf = (
),
ms: call.start ?? index,
},
end:
call.end === null
? undefined
: parseOffsetDateTimeInstant(
new Date(call.end ?? index).toISOString(),
),
endMs: call.end === null ? undefined : (call.end ?? index),
},
}));
@@ -505,6 +511,7 @@ describe("read-only decisions over many calls", () => {
),
ms: index,
},
end: parseOffsetDateTimeInstant(new Date(index).toISOString()),
endMs: index,
},
};
@@ -573,6 +580,7 @@ describe("read-only decisions over a long run of calls", () => {
),
ms: index,
},
end: parseOffsetDateTimeInstant(new Date(index).toISOString()),
endMs: index,
},
};
@@ -641,6 +649,7 @@ describe("read-only decisions over many lines", () => {
),
ms: index,
},
end: parseOffsetDateTimeInstant(new Date(index).toISOString()),
endMs: index,
},
};
@@ -24,6 +24,10 @@ const record = (
instant: parseOffsetDateTimeInstant(new Date(start).toISOString()),
ms: start,
},
end:
endMs === undefined
? undefined
: parseOffsetDateTimeInstant(new Date(endMs).toISOString()),
endMs,
},
});
@@ -31,6 +31,8 @@ export interface StateRecordFacts {
readonly start:
| { readonly instant: OffsetDateTimeInstant; readonly ms: number }
| undefined;
/** Exact completion, independent of the millisecond chronology policy. */
readonly end: OffsetDateTimeInstant | undefined;
readonly endMs: number | undefined;
};
}
@@ -630,19 +632,22 @@ export const replaceAt = (
const matchingKeys = (object: JsonObject, value: string): string[] =>
Object.keys(object).filter((key) => scalarText(object[key]) === value);
/** The entity object an item response carries with the keys repeating its path value: at the root or in one wrapper member, else as the element of a one-element array there. */
export const locateItem = (
/** Candidate entity objects at the first matching envelope tier. A known identity field constrains admission before choosing a tier; multiple candidates remain ambiguous. */
export const locateItems = (
body: JsonValue,
value: string,
): { readonly pointer: string; readonly keys: string[] } | undefined => {
idField?: string,
): readonly { readonly pointer: string; readonly keys: string[] }[] => {
const located = (
object: JsonValue | undefined,
pointer: string,
named = false,
) => {
const keys = isObject(object)
? matchingKeys(object, value).filter(
(key) => !named || IDENTITY_NAMES.includes(key),
? matchingKeys(object, value).filter((key) =>
idField !== undefined
? key === idField
: !named || IDENTITY_NAMES.includes(key),
)
: [];
return keys.length > 0 ? [{ pointer, keys }] : [];
@@ -658,8 +663,17 @@ export const locateItem = (
located(soleElement(item), `${pointerTo(name)}/0`, true),
),
];
const chosen = tiers.find((tier) => tier.length > 0);
return chosen?.length === 1 ? chosen[0] : undefined;
return tiers.find((tier) => tier.length > 0) ?? [];
};
/** The unique entity object at the first matching envelope tier; absent or ambiguous otherwise. */
export const locateItem = (
body: JsonValue,
value: string,
idField?: string,
): { readonly pointer: string; readonly keys: string[] } | undefined => {
const candidates = locateItems(body, value, idField);
return candidates.length === 1 ? candidates[0] : undefined;
};
/** Whether every item carries `name` as a distinct string or number. */
@@ -12,7 +12,7 @@ import {
scenarioCaptureFixture,
} from "../../../tests/fixtures/browserScenarioCapture.js";
import type { Evidence } from "../evidence.js";
import type { JsonValue } from "../jsonValue.js";
import { ownedJsonObjectSchema, type JsonValue } from "../jsonValue.js";
import { webNetworkCaptureSchema } from "../webNetworkCapture.js";
import {
analysisViewResultSchema,
@@ -842,7 +842,7 @@ it("leaves every line checked rather than compare more lines than the variant pa
path: "/api/items",
query: `?view=${String(ordinal)}`,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
}));
const shape = {
entities: new Map(
@@ -869,7 +869,7 @@ it("tells apart member names that hold the separator, so a line spelling them ot
path: "/api/items/1",
query,
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
});
const observed = (ordinal: number, object: JsonObject) => ({
position: ordinal,
@@ -909,7 +909,7 @@ it("leaves every line checked when comparing many mostly empty lines would excee
method: "GET",
path: "/api/items",
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
};
return Object.defineProperty(record, "query", {
enumerable: true,
@@ -950,7 +950,7 @@ const expandedItemLines = (items: number) => {
path: `/api/items/${String(ordinal % items)}`,
query: ordinal < items ? undefined : "?expand=shelf",
status: 200,
timing: { start: undefined, endMs: undefined },
timing: { start: undefined, end: undefined, endMs: undefined },
}),
);
const shapes = new Map(
@@ -1144,6 +1144,7 @@ it("finds each span's earliest write without scanning the writes before it", ()
return start;
},
},
end: offsetDateTimeInstantFromEpochMilliseconds(end),
get endMs() {
reads += 1;
return end;
@@ -2020,3 +2021,168 @@ it.each([
if (missingDuration) expect(state.write_clock).toBeNull();
},
);
it.each(["not a URL", "", 42, undefined])(
"bounds initial state at a successful write whose URL is unavailable: %j",
(url) => {
const source = captureOf([
["GET", "/api/items/1", { id: 1, title: "initial" }],
["PUT", "/api/items/1", { id: 1 }],
["GET", "/api/items/1", { id: 1, title: "after" }],
]);
const capture = webNetworkCaptureSchema.parse(source.normalizedResult);
const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported);
const request = ownedJsonObjectSchema.parse(reported.request);
if (url === undefined) delete request.url;
else request.url = url;
const before = structuredClone(capture);
const state = stateOf(parentOf(analysisViewCaptureEvidence(capture)));
expect(state.types).toMatchObject([
{ init_before_ordinal: 1, volatile_fields: [] },
]);
expect(state.entities).toMatchObject([
{ fields: { title: "initial" }, record_ordinals: [0] },
]);
expect(state.limitations).toContainEqual(
expect.stringMatching(/1.*writes.*unusable URL.*target is unknown/iu),
);
expect(
state.limitations.some((text) => text.includes("redacted URL")),
).toBe(false);
expect(capture).toEqual(before);
},
);
it("commits neither a manifest line nor state facts when inherited state admission fails", () => {
const records = webNetworkCaptureSchema.parse(
environmentManifestCaptureFixture([
{
startedDateTime: "2026-10-20T09:00:00Z",
time: 0,
request: { method: "GET", url: "https://manifest.test/api/health" },
response: {
status: 200,
content: { mimeType: "application/json", text: '{"healthy":true}' },
},
},
{
startedDateTime: "x".repeat(5000),
time: 0,
request: { method: "GET", url: "https://manifest.test/api/items" },
response: {
status: 200,
content: { mimeType: "application/json", text: "[]" },
},
},
]),
).records;
const accumulator = new EnvironmentStateAccumulator(32 * 1024);
expect(accumulator.add(records[0]!).ok).toBe(true);
expect(accumulator.add(records[1]!).ok).toBe(false);
const admitted = accumulator.finish();
if (!admitted.ok) throw admitted.error;
expect(admitted.value.records.map(({ ordinal }) => ordinal)).toEqual([0]);
const retry = structuredClone(records[1]!);
if (
retry.reported === null ||
typeof retry.reported !== "object" ||
Array.isArray(retry.reported)
)
throw new Error("Expected fixture object");
retry.reported.startedDateTime = "2026-10-20T09:00:01Z";
expect(accumulator.add(retry).ok).toBe(true);
const state = accumulator.finishState();
if (!state.ok) throw state.error;
expect(state.value.documents.map(({ path }) => path)).toEqual([
"/api/health",
"/api/items",
]);
});
it.each([
{ method: "GET", status: 200 },
{ method: "PUT", status: 500 },
])(
"does not infer a successful state write from an excluded $method/$status record",
({ method, status }) => {
const source = captureOf([
["GET", "/api/items/1", { id: 1, title: "initial" }],
[method, "/api/items/1", {}, status],
["GET", "/api/items/1", { id: 1, title: "after" }],
]);
const capture = webNetworkCaptureSchema.parse(source.normalizedResult);
const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported);
ownedJsonObjectSchema.parse(reported.request).url = "not a URL";
const state = stateOf(parentOf(analysisViewCaptureEvidence(capture)));
expect(state.types).toMatchObject([
{ init_before_ordinal: null, volatile_fields: ["title"] },
]);
expect(state.entities).toMatchObject([{ fields: { title: "after" } }]);
expect(
state.limitations.some((text) =>
text.includes("writes had an unusable URL"),
),
).toBe(false);
},
);
it.each([{}, { status: "unknown" }, { status: null }])(
"keeps a possible unlocated write when its response status is unknown: %j",
(response) => {
const source = captureOf([
["GET", "/api/items/1", { id: 1, title: "initial" }],
["PUT", "/api/items/1", {}],
["GET", "/api/items/1", { id: 1, title: "after" }],
]);
const capture = webNetworkCaptureSchema.parse(source.normalizedResult);
const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported);
ownedJsonObjectSchema.parse(reported.request).url = "not a URL";
reported.response = ownedJsonObjectSchema.parse(response);
const state = stateOf(parentOf(analysisViewCaptureEvidence(capture)));
expect(state.types).toMatchObject([
{ init_before_ordinal: 1, volatile_fields: [] },
]);
expect(state.entities).toMatchObject([
{ fields: { title: "initial" }, record_ordinals: [0] },
]);
},
);
it.each([true, false])(
"derives timeline limitations from the current state snapshot (timed: %s)",
(timed) => {
const accumulator = new EnvironmentStateAccumulator();
const empty = accumulator.finishState();
if (!empty.ok) throw empty.error;
expect(empty.value.timeline).toBe("record-order");
const record = webNetworkCaptureSchema.parse(
environmentManifestCaptureFixture([
{
...(timed ? { startedDateTime: "2026-10-20T09:00:00Z" } : {}),
time: 0,
request: { method: "GET", url: "https://snapshot.test/api/items" },
response: {
status: 200,
content: { mimeType: "application/json", text: "[]" },
},
},
]),
).records[0]!;
expect(accumulator.add(record).ok).toBe(true);
const current = accumulator.finishState();
if (!current.ok) throw current.error;
expect(current.value.timeline).toBe(
timed ? "recorded-start-time" : "record-order",
);
expect(
current.value.limitations.some((text) =>
text.includes("Some records carry no start time"),
),
).toBe(!timed);
expect(
current.value.limitations.some((text) =>
text.includes("No retained record had a parseable origin"),
),
).toBe(false);
},
);
@@ -195,6 +195,34 @@ describe("item actions", () => {
});
});
it("locates an action's entity by its known id field before response metadata", () => {
const response = { request_id: 1, data: ARCHIVED };
expect(
planOf(
[
[
"GET",
"/api/books",
[shelved(FIRST_BOOK, false), shelved(SECOND_BOOK, false)],
],
["GET", "/api/books/1", { data: shelved(FIRST_BOOK, false) }],
["GET", "/api/books/2", { data: shelved(SECOND_BOOK, false) }],
["POST", "/api/books/1/archive", response],
["GET", "/api/books/1", { data: ARCHIVED }],
["POST", "/api/books/1/archive", response],
["GET", "/api/books/1", { data: ARCHIVED }],
],
"post_books_archive_by_id",
),
).toMatchObject({
effect: "update",
served: true,
address: { by: "action", segment: "archive" },
response: { shape: "entity", pointer: "/data" },
rules: [{ field: "archived", rule: "set", value: true }],
});
});
it("serves no action seen changing its field only once, which may toggle it", () => {
expect(
planOf(
@@ -18,6 +18,7 @@ import { entityTarget, succeeded } from "./environmentStateWriteChecks.js";
import type { WriteIo } from "./environmentStateWriteEffects.js";
import {
pathValuesOf,
pathParameterKeys,
type BodyAddress,
type StateWritePlan,
type WriteAddress,
@@ -95,19 +96,20 @@ export const itemAction = (
)
return undefined;
const views = facts.itemViews(target);
const itemKey = pathParameterKeys(tool.path_template).get(
segments.length - 2,
)!;
const showsEntity = (record: StateRecordFacts): boolean => {
const id = pathValuesOf(tool.path_template, record.path)?.get(
item.slice(1, -1),
);
const id = pathValuesOf(tool.path_template, record.path)?.get(itemKey);
if (id === undefined || record.status === 201) return false;
if (facts.emptyResponse(record.ordinal)) return true;
return facts.withBody(record.ordinal, (body) => {
if (body === undefined || body === null) return true;
const located = locateItem(body, id);
const located = locateItem(body, id, idField);
const object =
located === undefined ? undefined : at(body, located.pointer);
return (
located?.keys.includes(idField) === true &&
located !== undefined &&
isObject(object) &&
views.some(
(view) =>
@@ -152,7 +152,7 @@ export class FailedReadBlame {
const response =
call === undefined
? undefined
: this.#responseObject(records[call]!, item.entity.id);
: this.#responseObject(records[call]!, item.entity);
if (response !== undefined) responses.set(name, response);
}
return fieldBlame(
@@ -169,10 +169,13 @@ export class FailedReadBlame {
/** The entity a write's response shows, located by its id. */
#responseObject(
record: StateRecordFacts,
id: string,
entity: StateEntity,
): JsonObject | undefined {
return this.#facts.io.withBody(record.ordinal, (body) => {
const located = body === undefined ? undefined : locateItem(body, id);
const located =
body === undefined
? undefined
: locateItem(body, entity.id, this.#facts.idField(entity.type));
const object =
located === undefined ? undefined : at(body!, located.pointer);
return isObject(object) ? object : undefined;
@@ -119,6 +119,22 @@ it(
);
describe("coincidences no member derives a created field from", () => {
it.each([
["before", (value: string) => `𐐀${value}`],
["after", (value: string) => `${value}𐐀`],
])(
"keeps a member accepted when an astral letter joins its text %s",
(_, label) => {
const value = "harbour-notes";
const body = create(
{ id: 9, name: value, label: label(value) },
{ name: value },
);
expect(body.accepted).toEqual(["name"]);
expect(body.fixed).toEqual([]);
},
);
it("keeps a reference accepted when a constant repeats another reference of the named entity", () => {
const channel = "1bdxw3qoyj8zjyyhdjbdobdf4r";
const user = "c10c054bc2a4478bb78e0210549376b1";
@@ -8,7 +8,12 @@ import {
} from "./environmentStateRules.js";
import { GENERATED } from "./environmentStateWriteGenerated.js";
import type { FieldRule } from "./environmentStateWriteRules.js";
import { delimitedOffsets, readClock } from "./environmentStateWriteValues.js";
import {
delimitedOffsets,
joinsAfter,
joinsBefore,
readClock,
} from "./environmentStateWriteValues.js";
/** Shorter body strings are too common to show that a field copied one. */
export const COPY_MIN_LENGTH = 8;
@@ -52,12 +57,6 @@ export const storedMembers = (
}),
);
/** A letter or digit, which joins a token to the text around it. */
const WORD = /[\p{L}\p{N}]/u;
const joins = (character: string | undefined): boolean =>
character !== undefined && WORD.test(character);
/**
* The texts that hold each token delimited, read in one pass per text: from each start after a
* delimiter, each length a token has that a delimiter ends, rather than a search per token and text.
@@ -72,10 +71,10 @@ const holdersOf = (
const holders = new Map<string, Set<string>>();
for (const [field, text] of texts)
for (let at = 0; at < text.length; at += 1) {
if (joins(text[at - 1])) continue;
if (joinsBefore(text, at)) continue;
for (const length of lengths) {
if (at + length > text.length) break;
if (joins(text[at + length])) continue;
if (joinsAfter(text, at + length)) continue;
const token = text.slice(at, at + length);
if (tokens.has(token))
holders.set(token, (holders.get(token) ?? new Set()).add(field));
@@ -18,6 +18,7 @@ const write = (path: string, ordinal: number): StateRecordFacts => ({
instant: offsetDateTimeInstantFromEpochMilliseconds(ordinal),
ms: ordinal,
},
end: offsetDateTimeInstantFromEpochMilliseconds(ordinal),
endMs: ordinal,
},
});
@@ -0,0 +1,118 @@
import { expect, it } from "vitest";
import {
timedCaptureEvidence,
type TimedCaptureRow,
} from "../../../tests/fixtures/analysisView.js";
import {
FIRST_BOOK,
environmentStateViewOf,
} from "../../../tests/fixtures/environmentStateWrites.js";
it.each([
{ fraction: "123456790", served: true, label: "one nanosecond later" },
{ fraction: "123456788", served: false, label: "one nanosecond earlier" },
{ fraction: "123456789", served: false, label: "the same instant" },
{ fraction: "124456789", served: true, label: "one millisecond later" },
])("checks an update clock at $label", ({ fraction, served }) => {
const first = {
...FIRST_BOOK,
name: "Second title",
updated_at: "2026-10-17T18:00:03.123456789Z",
};
const second = {
...first,
name: "Third title",
updated_at: `2026-10-17T18:00:03.${fraction}Z`,
};
const state = environmentStateViewOf(
timedCaptureEvidence([
["GET", "/api/books", [FIRST_BOOK]],
["GET", "/api/books/1", FIRST_BOOK],
["PATCH", "/api/books/1", first, 200, { name: first.name }],
["GET", "/api/books/1", first],
["PATCH", "/api/books/1", second, 200, { name: second.name }],
["GET", "/api/books/1", second],
]),
);
expect(
state.writes.find((plan) => plan.tool === "patch_books_by_id"),
).toMatchObject({
served,
reason: served ? null : "unexplained-change",
checked_record_ordinals: served ? [3, 5] : [3],
rules: expect.arrayContaining([
{
field: "updated_at",
rule: "clock",
format: { kind: "iso", separator: "T", fraction: 9, zone: "Z" },
},
]),
});
});
it.each([false, true])(
"learns only after an unknown entity is observed again (update while unknown: %s)",
(updateWhileUnknown) => {
const initial = { ...FIRST_BOOK, revision: 1 };
const observed = { ...initial, name: "Second title", revision: 2 };
const final = { ...observed, name: "Third title" };
const unknownCall: TimedCaptureRow[] = updateWhileUnknown
? [["PATCH", "/api/books/1", observed, 200, { name: observed.name }]]
: [];
const state = environmentStateViewOf(
timedCaptureEvidence([
["GET", "/api/books", [initial]],
["GET", "/api/books/1", initial],
["POST", "/api/books/1/import", { queued: true }, 302, {}],
...unknownCall,
["GET", "/api/books/1", observed],
["PATCH", "/api/books/1", final, 200, { name: final.name }],
["GET", "/api/books/1", final],
]),
);
expect(
state.writes.find((plan) => plan.tool === "patch_books_by_id"),
).toMatchObject({
served: true,
reason: null,
exemplar_ordinal: updateWhileUnknown ? 5 : 4,
checked_record_ordinals: [updateWhileUnknown ? 6 : 5],
rules: [{ field: "name", rule: "echo" }],
});
},
);
it.each([
{ requestId: 1, ambiguous: false, served: true },
{ requestId: 2, ambiguous: false, served: true },
{ requestId: 1, ambiguous: true, served: false },
])(
"uses the known identity field for response observation ($requestId, ambiguous: $ambiguous)",
({ requestId, ambiguous, served }) => {
const changed = { ...FIRST_BOOK, name: "Second title" };
const response = {
request_id: requestId,
data: changed,
...(ambiguous ? { other: changed } : {}),
};
const state = environmentStateViewOf(
timedCaptureEvidence([
["GET", "/api/books", [FIRST_BOOK]],
["GET", "/api/books/1", FIRST_BOOK],
["PATCH", "/api/books/1", response, 200, { name: changed.name }],
["GET", "/api/books/1", changed],
]),
);
const plan = state.writes.find(
(candidate) => candidate.tool === "patch_books_by_id",
);
expect(plan?.served).toBe(served);
if (served)
expect(plan).toMatchObject({
observation: { ordinal: 2, pointer: "/data" },
response: { ordinal: 2, shape: "entity", pointer: "/data" },
checked_record_ordinals: [3],
});
},
);
@@ -28,6 +28,7 @@ import {
type SimulatedState,
} from "./environmentStateWriteChecks.js";
import {
compareValues,
explainUpdate,
responseShapeOf,
} from "./environmentStateWriteExplain.js";
@@ -367,11 +368,12 @@ const updatedObservation = (
step.record.ordinal,
(body): Observed | undefined => {
const located =
body === undefined ? undefined : locateItem(body, entity.id);
body === undefined
? undefined
: locateItem(body, entity.id, entity.idField);
const object =
located === undefined ? undefined : at(body!, located.pointer);
return located?.keys.includes(entity.idField) === true &&
isObject(object)
return located !== undefined && isObject(object)
? {
ordinal: step.record.ordinal,
pointer: located.pointer,
@@ -657,6 +659,9 @@ export const updateEntity = (
return UNKNOWN;
}
const { targets, target, recorded } = outcomes(context, entity, after.object);
// An unknown baseline cannot establish this call's changes. Keep its outcome
// without creating or refining rules from values another write may have changed.
if (context.unknown(type, collection, id)) return recorded();
// Only confirmed absence makes a bodyless action; malformed or unretained content stays unknown.
const body =
"body" in named
@@ -670,7 +675,6 @@ export const updateEntity = (
fail(draft, step, "body-unusable");
return recorded();
}
const unknown = context.unknown(type, collection, id);
const call = { ...callOf(step, body), body };
const fitted = fitUpdateClocks(
draft,
@@ -704,7 +708,7 @@ export const updateEntity = (
const explained = { ...fitted.explained, rules: later };
for (const member of keptOrders(call.body, after.object))
draft.kept.add(member);
if (explained.unexplained.length > 0 && !unknown) {
if (explained.unexplained.length > 0) {
refuse(draft, step, "unexplained-change");
return recorded();
}
@@ -713,16 +717,15 @@ export const updateEntity = (
searchFrom(context, step),
itemReads(context, { type, idField, collection, id }),
)?.object;
// Each call must agree with the plan; one on an unknown entity cannot be checked, so it adds nothing to the plan.
// Each call with a known baseline must agree with the plan.
if (
built(draft) &&
(unknown ||
contradictsPlan(
{ ...draft, plan: { ...draft.plan, rules } },
entity.fields,
after.object,
explained,
) ||
(contradictsPlan(
{ ...draft, plan: { ...draft.plan, rules } },
entity.fields,
after.object,
explained,
) ||
// A member calls left out is cleared by every one or by none.
// A reference this call or an earlier one showed counts its member as the field's, whichever came first.
defaultsDisagree(explained.body.defaults, { earlier: draft.updates }, [
@@ -735,7 +738,7 @@ export const updateEntity = (
[...rules, ...explained.rules],
))
) {
if (!unknown) refuse(draft, step, "unexplained-change");
refuse(draft, step, "unexplained-change");
return recorded();
}
draft.plan.rules = rules;
@@ -778,9 +781,10 @@ export const updateEntity = (
after: after.object,
kept,
});
if (!written.ok || (disagrees && !unknown))
if (!written.ok || disagrees) {
refuse(draft, step, "unexplained-change");
if (!written.ok || disagrees || unknown) return recorded();
return recorded();
}
rememberCall(
context.io.retain,
draft,
@@ -893,7 +897,7 @@ const contradictsGeneratedClock = (
candidate.field === rule.field && candidate.rule === "clock",
) ||
next === undefined ||
next.ms <= previous.ms
compareValues(after[rule.field], ownMember(before, rule.field))! <= 0
);
});
@@ -13,6 +13,7 @@ import {
type CreateEvidence,
} from "./environmentStateWriteExplain.js";
import type { UpdateEvidence } from "./environmentStateWriteChanges.js";
import { writtenResponse } from "./environmentStateWriteApply.js";
import {
callRenewal,
unexplainedRenewals,
@@ -504,6 +505,35 @@ it.each<[string, string[]]>([
type Shaped = [WriteResponse["shape"], string | null];
it.each([false, true])(
"keeps response entity selection distinct from repeated envelope metadata, ambiguous=%s",
(ambiguous) => {
const recorded = {
request_id: "42",
data: { id: "42", title: "Captured title" },
...(ambiguous ? { other: { id: "42", title: "Another entity" } } : {}),
};
const shape = responseShapeOf(3, recorded, false, {
idField: "id",
id: "42",
});
expect(
writtenResponse(shape, recorded, {
id: "43",
fields: { id: "43", title: "Authored title" },
}),
).toEqual({
ok: true,
value: ambiguous
? recorded
: {
request_id: "42",
data: { id: "43", title: "Authored title" },
},
});
},
);
it.each<[string, JsonValue | undefined, boolean, Shaped]>([
["no body", undefined, true, ["empty", null]],
["an empty response with a body", { id: "42" }, true, ["empty", null]],
@@ -4,7 +4,7 @@ import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js
import {
ownMember,
isObject,
locateItem,
locateItems,
pointerTo,
same,
scalarText,
@@ -174,7 +174,7 @@ const requestRule = (
index: BodyIndex,
field: string,
value: JsonValue,
): FieldRule | undefined => {
): FieldRule | "ambiguous-path" | undefined => {
const { body, pathValues } = evidence.call;
if (Object.hasOwn(body, field) && same(body[field], value))
return { field, rule: "echo" };
@@ -185,9 +185,11 @@ const requestRule = (
if (from !== undefined) return { field, rule: "copy", from };
}
const text = scalarText(value);
for (const [name, pathValue] of pathValues)
if (text !== undefined && text === pathValue)
return { field, rule: "path", from: name };
const paths = [...pathValues].filter(
([, value]) => text !== undefined && text === value,
);
if (paths.length > 1) return "ambiguous-path";
if (paths.length === 1) return { field, rule: "path", from: paths[0]![0] };
const slugged = typeof value === "string" && value.length >= 3;
const source = slugged ? index.slugs.get(value) : undefined;
return source === undefined ? undefined : slugRule(field, source);
@@ -357,8 +359,14 @@ export const explainCreate = (
const index = indexBody(evidence.call.body);
// The member each slug the held entities' `&` spellings refused was slugged from.
const refusedSlugs = new Map<string, string>();
let ambiguousPath = false;
for (const [field, value] of Object.entries(evidence.entity)) {
const requested = requestRule(evidence, index, field, value);
if (requested === "ambiguous-path") {
ambiguousPath = true;
unexplained.push(field);
continue;
}
const held = heldSlugRule(requested, evidence.call.body, evidence.type);
if (requested?.rule === "slug" && held === undefined)
refusedSlugs.set(field, requested.from);
@@ -468,12 +476,14 @@ export const explainCreate = (
rules,
body: blank.length === 0 ? shaped : { ...shaped, blank },
// A template rebuilds only its generated value, so other text it keeps ties it like a constant.
tied: [
...[...constants, ...templates].map((field) =>
ownMember(evidence.entity, field)!,
),
...literals,
].some((text) => tokens.some((token) => containsDelimited(text, token))),
tied:
ambiguousPath ||
[
...[...constants, ...templates].map((field) =>
ownMember(evidence.entity, field)!,
),
...literals,
].some((text) => tokens.some((token) => containsDelimited(text, token))),
};
};
@@ -646,8 +656,11 @@ export const responseShapeOf = (
): WriteResponse => {
if (empty) return { ordinal, shape: "empty", pointer: null };
if (body === undefined) return { ordinal, shape: "recorded", pointer: null };
const located = locateItem(body, entity.id);
if (located?.keys.includes(entity.idField) === true)
const candidates = locateItems(body, entity.id, entity.idField);
if (candidates.length > 1)
return { ordinal, shape: "recorded", pointer: null };
const located = candidates[0];
if (located !== undefined)
return { ordinal, shape: "entity", pointer: located.pointer };
if (scalarText(body) === entity.id)
return { ordinal, shape: "identity", pointer: "" };
@@ -22,6 +22,7 @@ const record = (
instant: parseOffsetDateTimeInstant(new Date(start).toISOString()),
ms: start,
},
end: parseOffsetDateTimeInstant(new Date(end).toISOString()),
endMs: end,
},
});
@@ -68,7 +69,7 @@ describe("short-id creation limits", () => {
const prior = record(0, "GET", "/api/projects", [0, 5]);
const unknown = {
...prior,
timing: { ...prior.timing, endMs: undefined },
timing: { ...prior.timing, end: undefined, endMs: undefined },
};
const post = record(1, "POST", "/api/imports", [10, 11]);
const later = record(2, "GET", "/api/projects/7", [20, 21]);
@@ -23,6 +23,7 @@ import {
import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js";
import {
pathValuesOf,
pathParameterKeys,
type FieldRule,
type WriteEffect,
} from "./environmentStateWriteRules.js";
@@ -414,13 +415,9 @@ export const anyLinkedSource = (
plan.collection !== null &&
normalizedPath(plan.collection) === plan.target &&
!segmentsOf(plan.target).some(isParam) &&
segmentsOf(template)
.filter(isParam)
.every((segment) =>
plan.rules.some(
(rule) => rule.rule === "path" && rule.from === segment.slice(1, -1),
),
);
[...pathParameterKeys(template).values()].every((name) =>
plan.rules.some((rule) => rule.rule === "path" && rule.from === name),
);
/**
* Whether a linked create's call sends the path values its exemplar did, each as recorded or in the
@@ -407,6 +407,23 @@ export interface WriteCall {
readonly body: JsonObject | undefined;
}
/** Parameter lookup keys preserve repeated placeholder occurrences by segment position. */
export const pathParameterKeys = (template: string): Map<number, string> => {
const parameters = segmentsOf(template).flatMap((segment, index) =>
isParam(segment) ? [[index, segment.slice(1, -1)] as const] : [],
);
const repeated =
new Set(parameters.map(([, name]) => name)).size !== parameters.length;
// If any name repeats, encode every key so a literal name cannot collide with
// an indexed key. Unique-name templates retain their established lookup keys.
return new Map(
parameters.map(([index, name]) => [
index,
repeated ? JSON.stringify([index, name]) : name,
]),
);
};
/** The value each template parameter takes in a concrete path, decoded; undefined when the path does not instantiate it. */
export const pathValuesOf = (
template: string,
@@ -416,13 +433,14 @@ export const pathValuesOf = (
const concrete = segmentsOf(path);
if (pattern.length !== concrete.length) return undefined;
const values = new Map<string, string>();
const keys = pathParameterKeys(template);
for (const [index, segment] of pattern.entries()) {
const value = concrete[index]!;
if (!isParam(segment)) {
if (segment !== value) return undefined;
continue;
}
values.set(segment.slice(1, -1), decodedSegment(value));
values.set(keys.get(index)!, decodedSegment(value));
}
return values;
};
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import type { JsonValue } from "../jsonValue.js";
import { jsonObjectSchema, type JsonValue } from "../jsonValue.js";
import { webNetworkCaptureSchema } from "../webNetworkCapture.js";
import { EnvironmentStateAccumulator } from "./EnvironmentStateAccumulator.js";
import {
@@ -86,6 +86,43 @@ it("retains learned clock rules but withholds their serving when the global cloc
).toEqual(plan);
});
it.each([
["000000000", 0, "2026-10-17T18:00:12.000Z"],
["000001", 0, "2026-10-17T18:00:13.000Z"],
["000000001", 0, "2026-10-17T18:00:13.000Z"],
["123000000", 0, "2026-10-17T18:00:13.000Z"],
["999999", 0.001, "2026-10-17T18:00:13.000Z"],
["999999", 0.001001, "2026-10-17T18:00:14.000Z"],
])(
"starts the served clock after a final captured fractional instant .%s plus %sms",
(fraction, duration, expected) => {
const evidence = timedCaptureEvidence([
...BOOK_ROWS,
["GET", "/api/health", { ready: true }, 200, undefined, 0],
]);
const capture = webNetworkCaptureSchema.parse(evidence.normalized_result);
const state = environmentStateViewOf({
...evidence,
normalized_result: {
...capture,
records: capture.records.map((record, index) =>
index === capture.records.length - 1
? {
...record,
reported: {
...jsonObjectSchema.parse(record.reported),
startedDateTime: `2026-10-17T18:00:12.${fraction}Z`,
time: duration,
},
}
: record,
),
},
});
expect(state.write_clock).toBe(expected);
},
);
it.each(["delete", "mismatched-query"])(
"does not parse unused request content for %s under a bounded state allowance",
(kind) => {
@@ -2,6 +2,10 @@ import type { ExtractedState } from "./environmentStateRules.js";
import { EnvironmentStateWriteSimulation } from "./EnvironmentStateWriteSimulation.js";
import type { WriteSimulationInput } from "./environmentStateWriteEffects.js";
import { usesWriteClock } from "./environmentStateWriteRules.js";
import {
compareOffsetDateTimeInstants,
type OffsetDateTimeInstant,
} from "../offsetDateTime.js";
/** The latest end a record can have and still round up to a whole second a date can hold. */
const LATEST_END_MS = 8_640_000_000_000_000 - 1000;
@@ -13,17 +17,23 @@ const clockStart = (
// The clock is global: any unknown completion may extend beyond every known end.
if (records.some((record) => record.timing.endMs === undefined)) return null;
// A record timed past the range of a date carries no usable time, so the clock starts from the others.
const end = records.reduce(
(latest, record) =>
record.timing.endMs !== undefined &&
Math.abs(record.timing.endMs) <= LATEST_END_MS
? Math.max(latest, record.timing.endMs)
: latest,
Number.NEGATIVE_INFINITY,
);
return Number.isFinite(end)
? new Date(Math.ceil(end / 1000) * 1000).toISOString()
: null;
let latest: OffsetDateTimeInstant | undefined;
for (const record of records) {
if (
record.timing.endMs === undefined ||
Math.abs(record.timing.endMs) > LATEST_END_MS
)
continue;
const end = record.timing.end;
if (end === undefined) return null;
if (latest === undefined || compareOffsetDateTimeInstants(end, latest) > 0)
latest = end;
}
return latest === undefined
? null
: new Date(
(latest.epochSecond + (latest.fraction === "" ? 0 : 1)) * 1000,
).toISOString();
};
/**
@@ -44,6 +44,14 @@ export type RecordedResponseDeclaration = ReturnType<
typeof readRecordedResponseDeclaration
>;
/** HAR's zero status sentinel and nonnumeric declarations do not prove a response status. */
export const recordedResponseStatus = (
declaration: RecordedResponseDeclaration,
): number | null => {
const status = declaration?.status;
return typeof status === "number" && status !== 0 ? status : null;
};
/** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */
const strictUtf8 = (): TextDecoder =>
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
@@ -83,6 +91,28 @@ export const readRecordedResponseBody = (
): RecordedResponseBody => {
const content = declaration?.content;
if (content == null) return { bodyText: null };
const bodyCoordinates = [
"/response/content/text",
"/response/content/encoding",
];
const within = (pointer: string, ancestor: string): boolean =>
pointer.startsWith(ancestor) &&
(pointer.length === ancestor.length || pointer[ancestor.length] === "/");
// Explicit value exclusions also govern imported retained facts. Keep their
// source text in complete Evidence, but never promote it into replay/state.
if (
record.redactions.some(
(redaction) =>
!("scope" in redaction) &&
bodyCoordinates.some((pointer) => within(pointer, redaction.pointer)),
) ||
record.binary_fields.some(
(field) =>
field.state === "redacted" &&
bodyCoordinates.some((pointer) => within(pointer, field.pointer)),
)
)
return { bodyText: null };
const { text, encoding } = content;
if (typeof text !== "string") return { bodyText: null };
if (encoding === undefined) {
@@ -3,7 +3,11 @@ import { createHash } from "node:crypto";
import { createEvidence } from "../evidence.js";
import { freezeJsonSnapshot } from "../immutableJson.js";
import { webNetworkCaptureSchema } from "../webNetworkCapture.js";
import { projectAnalysisView } from "./analysisView.js";
import {
projectAnalysisView,
analysisViewRequestSchema,
} from "./analysisView.js";
import { timedCaptureEvidence } from "../../../tests/fixtures/analysisView.js";
const capture = () => {
const body = Buffer.from("original body");
@@ -800,18 +804,6 @@ it.each([
"bytes that are not its text",
{ binaryFields: [retainedBytes("/request/path", "/b")] },
],
[
"a byte count that is not its text",
{ binaryFields: [{ ...retainedBytes("/request/path", "/a"), bytes: 3 }] },
],
[
"a digest that is not its text",
{
binaryFields: [
{ ...retainedBytes("/request/path", "/a"), sha256: "0".repeat(64) },
],
},
],
] as const)(
"keeps a mitmproxy path with %s out of the preview",
(_label, facets) => {
@@ -1040,3 +1032,111 @@ it.each(["scheme", "host", "port", "authority"])(
expect(JSON.stringify(reported)).not.toContain("password");
},
);
it.each(["base64", "bytes", "sha256", "duplicate"] as const)(
"refuses inconsistent retained binary %s at parent admission",
(fault) => {
const { parent, report } = capture();
const record = report.records[0]!;
const binary = record.binary_fields[0]!;
if (binary.state !== "retained")
throw new Error("Expected retained source bytes");
const changed = {
...binary,
...(fault === "base64"
? { content_base64: "!!!!" }
: fault === "bytes"
? { bytes: binary.bytes + 1 }
: { sha256: "0".repeat(64) }),
};
const invalid = {
...report,
records: [
{
...record,
binary_fields: fault === "duplicate" ? [binary, binary] : [changed],
},
],
};
const projected = projectAnalysisView(
{ ...parent, normalizedResult: invalid },
{ kind: "capture", detail: "preview", offset: 0, limit: 1 },
);
expect(projected).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError" },
});
},
);
it.each([
"/response/content/text",
"/response/content",
"/response",
"",
"binary-redaction",
"control",
])(
"respects retained response-body authority without altering raw facts: %s",
(marker) => {
const seed = timedCaptureEvidence([
["GET", "/api/items", [{ id: 1, name: "withheld-value" }]],
]);
const report = webNetworkCaptureSchema.parse(seed.normalized_result);
const record = report.records[0]!;
if (marker === "binary-redaction")
record.binary_fields.push({
pointer: "/response/content/text",
representation: "har-base64-content",
state: "redacted",
content_base64: null,
bytes: null,
sha256: null,
});
else if (marker !== "control")
record.redactions.push({
pointer: marker,
reason: "explicit-sensitive-value",
});
const evidence = createEvidence(
{
path: report.artifact.path,
sha256: report.artifact.sha256!,
format: "file",
},
report.decoder,
{
operation: seed.operation,
parameters: seed.parameters,
result: report,
},
);
const parent = {
predicateType: evidence.predicate_type,
evidenceId: evidence.evidence_id,
operation: evidence.operation,
normalizedResult: evidence.normalized_result,
subject: evidence.subject,
parameters: evidence.parameters,
limitations: evidence.limitations,
};
const state = projectAnalysisView(
parent,
analysisViewRequestSchema.parse({ kind: "environment-state" }),
);
if (!state.ok || state.value.kind !== "environment-state")
throw new Error("Expected retained state projection");
expect(
JSON.stringify(state.value.entities).includes("withheld-value"),
).toBe(marker === "control");
const complete = projectAnalysisView(parent, {
kind: "capture",
detail: "complete",
offset: 0,
limit: 1,
});
expect(
complete.ok && JSON.stringify(complete.value).includes("withheld-value"),
).toBe(true);
},
);
@@ -1,5 +1,8 @@
import { z } from "zod";
import { readRecordedResponseDeclaration } from "./recordedResponseBody.js";
import {
readRecordedResponseDeclaration,
recordedResponseStatus,
} from "./recordedResponseBody.js";
import {
parseRecordedTransportUrl,
parseUrlQueryParameterNames,
@@ -148,7 +151,6 @@ export const readRecordedRequestLine = (
return excluded("method-unavailable");
if (parsed.path === null) return excluded("url-unparseable");
const response = readRecordedResponseDeclaration(reported);
const status = response?.status;
const mime = response?.content?.mimeType;
const rawMime = typeof mime === "string" ? mime : "";
if (!temporary(textBytes(rawMime.length))) return null;
@@ -158,7 +160,7 @@ export const readRecordedRequestLine = (
origin: parsed.origin,
path: parsed.path,
query: parsed.query,
status: typeof status === "number" && status !== 0 ? status : null,
status: recordedResponseStatus(response),
rawMime,
};
};
+29
View File
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import { z } from "zod";
import {
addOffsetDateTimeMilliseconds,
inspectRecordedDateTime,
offsetDateTimeSchema,
readRecordedDateTime,
@@ -22,6 +23,34 @@ it.each([
expect(inspectRecordedDateTime(text).kind).toBe(kind);
});
it.each([
["000001", 0, 10, "000001"],
["999999", 0.001, 11, ""],
["999999", 0.001001, 11, "000000001"],
["1234567890123456789", 1000.001, 11, "1234577890123456789"],
["", 1e-7, 10, "0000000001"],
["", 999.9999999999999, 10, "9999999999999999"],
])(
"adds duration %s + %sms without losing decimal precision",
(fraction, duration, epochSecond, expectedFraction) => {
expect(
addOffsetDateTimeMilliseconds({ epochSecond: 10, fraction }, duration),
).toEqual({ epochSecond, fraction: expectedFraction });
},
);
it.each([-1, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_VALUE])(
"keeps unsupported recorded duration %s unknown",
(duration) => {
expect(
addOffsetDateTimeMilliseconds(
{ epochSecond: 10, fraction: "" },
duration,
),
).toBeUndefined();
},
);
const ACCEPTED: readonly (readonly [string, string])[] = [
["a UTC timestamp", "2026-10-09T04:00:00Z"],
["a positive offset", "2026-10-09T04:00:00+02:00"],
+42 -6
View File
@@ -146,12 +146,14 @@ export const offsetDateTimeInstantFromEpochMilliseconds = (
const millisecondsWithinSecond = epochMilliseconds - epochSecond * 1000;
return {
epochSecond,
fraction: fractionalSecondFromMilliseconds(millisecondsWithinSecond),
fraction: secondsFromMilliseconds(millisecondsWithinSecond).fraction,
};
};
const fractionalSecondFromMilliseconds = (milliseconds: number): string => {
if (milliseconds === 0) return "";
const secondsFromMilliseconds = (
milliseconds: number,
): { readonly seconds: number; readonly fraction: string } => {
if (milliseconds === 0) return { seconds: 0, fraction: "" };
const [coefficient = "", exponentText] = String(milliseconds)
.toLowerCase()
.split("e");
@@ -166,9 +168,43 @@ const fractionalSecondFromMilliseconds = (milliseconds: number): string => {
: decimalPosition < digits.length
? `${digits.slice(0, decimalPosition)}.${digits.slice(decimalPosition)}`
: `${digits}${"0".repeat(decimalPosition - digits.length)}`;
return (
fraction.includes(".") ? fraction.slice(fraction.indexOf(".") + 1) : ""
).replace(/0+$/u, "");
const separator = fraction.indexOf(".");
return {
seconds: Number(separator < 0 ? fraction : fraction.slice(0, separator)),
fraction: (separator < 0 ? "" : fraction.slice(separator + 1)).replace(
/0+$/u,
"",
),
};
};
/** Add a recorded nonnegative decimal millisecond duration without rounding away either fraction. */
export const addOffsetDateTimeMilliseconds = (
start: OffsetDateTimeInstant,
milliseconds: number,
): OffsetDateTimeInstant | undefined => {
if (!Number.isFinite(milliseconds) || milliseconds < 0) return undefined;
if (milliseconds === 0) return start;
const duration = secondsFromMilliseconds(milliseconds);
const wholeSeconds = start.epochSecond + duration.seconds;
if (!Number.isSafeInteger(wholeSeconds)) return undefined;
const length = Math.max(start.fraction.length, duration.fraction.length);
const digits: string[] = [];
let carry = 0;
for (let index = length - 1; index >= 0; index--) {
const sum =
(start.fraction.charCodeAt(index) || 48) -
48 +
(duration.fraction.charCodeAt(index) || 48) -
48 +
carry;
digits.push(String(sum % 10));
carry = Math.floor(sum / 10);
}
const epochSecond = wholeSeconds + carry;
return Number.isSafeInteger(epochSecond)
? { epochSecond, fraction: digits.reverse().join("").replace(/0+$/u, "") }
: undefined;
};
/** Compare instants exactly while treating absent fraction digits as zeroes. */
+105
View File
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { expect, it } from "vitest";
import {
freezeJsonSnapshot,
@@ -116,3 +117,107 @@ it("retains schema normalization when an owned source contains a hidden reported
Object.hasOwn(parsed.data.container.reported as object, "hidden"),
).toBe(false);
});
it.each([
{
name: "malformed base64",
content_base64: "!!!!",
bytes: 0,
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
},
{
name: "noncanonical pad bits",
content_base64: "AR==",
bytes: 1,
sha256: createHash("sha256")
.update(Buffer.from([1]))
.digest("hex"),
},
{
name: "mismatched byte count",
content_base64: "YQ==",
bytes: 2,
sha256: "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb",
},
{
name: "mismatched digest",
content_base64: "YQ==",
bytes: 1,
sha256: "0".repeat(64),
},
])(
"refuses $name in retained binary facts before projection",
({ name: _name, ...binary }) => {
const report = capture();
const record = {
...report.records[0]!,
binary_fields: [
{
pointer: "/response/content/text",
representation: "har-base64-content",
state: "retained",
...binary,
},
],
};
const parsed = webNetworkCaptureSchema.safeParse({
...report,
records: [record],
});
expect(parsed.success).toBe(false);
},
);
it.each([
{ name: "empty", bytes: Buffer.alloc(0) },
{ name: "Unicode", bytes: Buffer.from("雪�") },
{ name: "multiple windows", bytes: Buffer.alloc(100_001, 0xff) },
])(
"retains $name canonical binary identity across bounded digest windows",
({ bytes }) => {
const report = capture();
const source = {
...report,
records: [
{
...report.records[0]!,
binary_fields: [
{
pointer: "/body",
representation: "producer-bytes",
state: "retained",
content_base64: bytes.toString("base64"),
bytes: bytes.length,
sha256: createHash("sha256").update(bytes).digest("hex"),
},
],
},
],
};
const parsed = webNetworkCaptureSchema.parse(source);
expect(parsed).toEqual(source);
},
);
it("refuses multiple binary declarations for one source coordinate", () => {
const report = capture();
const binary = (text: string) => ({
pointer: "/response/content/text",
representation: "har-base64-content",
state: "retained",
content_base64: Buffer.from(text).toString("base64"),
bytes: Buffer.byteLength(text),
sha256: createHash("sha256").update(text).digest("hex"),
});
expect(
webNetworkCaptureSchema.safeParse({
...report,
records: [
{
...report.records[0]!,
binary_fields: [binary("first"), binary("second")],
},
],
}).success,
).toBe(false);
});
+46 -2
View File
@@ -1,3 +1,5 @@
import { createHash } from "node:crypto";
import { canonicalBase64Schema } from "./stringPatterns.js";
import { z } from "zod";
import {
browserDeadlineIncrease,
@@ -215,7 +217,35 @@ export const webNetworkCaptureBinaryDescriptorSchema = z.discriminatedUnion(
[retainedBinaryDescriptorSchema, redactedBinaryDescriptorSchema],
);
const binarySchema = z.discriminatedUnion("state", [
retainedBinaryDescriptorSchema.extend({ content_base64: z.string() }),
retainedBinaryDescriptorSchema
.extend({ content_base64: canonicalBase64Schema })
.superRefine((field, context) => {
const encoded = field.content_base64;
const decodedBytes = Buffer.byteLength(encoded, "base64");
if (field.bytes !== decodedBytes) {
context.addIssue({
code: "custom",
path: ["bytes"],
message:
"Retained binary byte count differs from its canonical content.",
});
return;
}
// Quartet-aligned windows authenticate every retained byte without
// allocating a second whole body while the capture remains live.
const hash = createHash("sha256");
for (let offset = 0; offset < encoded.length; offset += 65536)
hash.update(
Buffer.from(encoded.slice(offset, offset + 65536), "base64"),
);
if (hash.digest("hex") !== field.sha256)
context.addIssue({
code: "custom",
path: ["sha256"],
message:
"Retained binary SHA-256 differs from its canonical content.",
});
}),
redactedBinaryDescriptorSchema.extend({ content_base64: z.null() }),
]);
const numberSchema = z.strictObject({
@@ -244,7 +274,21 @@ export const webNetworkCaptureRecordSchema = z.strictObject({
}),
]),
reported: jsonValueSchema,
binary_fields: z.array(binarySchema),
binary_fields: z.array(binarySchema).superRefine((fields, context) => {
const pointers = new Set<string>();
for (const [index, field] of fields.entries()) {
if (pointers.has(field.pointer)) {
context.addIssue({
code: "custom",
path: [index, "pointer"],
message:
"Each retained source coordinate can declare binary facts only once.",
});
return;
}
pointers.add(field.pointer);
}
}),
numeric_literals: z.array(numberSchema),
redactions: z.array(redactionSchema),
limitations: z.array(z.string()),