mirror of
https://github.com/morluto/rea.git
synced 2026-11-07 05:56:19 -10:00
refactor(captures): consolidate recorded exchange body facts
This commit is contained in:
1 parent
c2b0c955cd
commit
0973649cec
14 files changed
+410
-226
No files matched your search
@@ -1,14 +1,16 @@
|
||||
import { basename } from "node:path";
|
||||
import { isIP } from "node:net";
|
||||
|
||||
import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js";
|
||||
import {
|
||||
isJsonLike,
|
||||
mediaTypeEssence,
|
||||
} from "../../domain/analysisView/recordedMediaType.js";
|
||||
import {
|
||||
decodeRetainedBase64,
|
||||
readRecordedResponseBody,
|
||||
type RecordedResponseBody,
|
||||
} from "../../domain/analysisView/recordedResponseBody.js";
|
||||
import { scenarioContentMediaType } from "../../domain/analysisView/scenarioCaptureRecords.js";
|
||||
import { mediaTypeEssence } from "../../domain/analysisView/webNetworkRouteTemplates.js";
|
||||
import type { BrowserScenarioEvent } from "../../domain/browserScenarioCaptureValues.js";
|
||||
import type { EnvironmentBuildReport } from "../../domain/environmentBundleExport.js";
|
||||
import {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
environmentStateServeFixture,
|
||||
} from "../../../tests/fixtures/environmentServe.js";
|
||||
import { compareEnvironment } from "./environmentCompare.js";
|
||||
import { requestBody } from "./environmentCompareProbe.js";
|
||||
import { REPLAY_BODY_LIMIT_BYTES } from "./environmentSource.js";
|
||||
|
||||
const responseCapture = (
|
||||
@@ -144,3 +145,24 @@ describe("environment comparison response representation", () => {
|
||||
expect(compare(json, text).probes[0]?.results[0]?.equal).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ mimeType: " TEXT/PLAIN ; charset=utf-8", expected: { value: 1 } },
|
||||
{ mimeType: "application/problem+json", expected: { value: 1 } },
|
||||
{ mimeType: "text/problem+json", expected: '{"value":1}' },
|
||||
])(
|
||||
"reads probe request arguments independently of body-size and parameter admission: $mimeType",
|
||||
({ mimeType, expected }) => {
|
||||
const capture = environmentManifestCaptureFixture([
|
||||
{
|
||||
request: {
|
||||
method: "POST",
|
||||
url: "http://app.example/api/value",
|
||||
bodySize: "not-a-size",
|
||||
postData: { mimeType, text: '{"value":1}', params: 7 },
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(requestBody(capture.records[0]!)).toEqual(expected);
|
||||
},
|
||||
);
|
||||
@@ -1,11 +1,15 @@
|
||||
import { z } from "zod";
|
||||
import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js";
|
||||
|
||||
import { readRecordedRequestBody } from "../../domain/analysisView/recordedExchangeBody.js";
|
||||
import {
|
||||
isJsonLike,
|
||||
mediaTypeEssence,
|
||||
} from "../../domain/analysisView/recordedMediaType.js";
|
||||
import {
|
||||
primaryOrigin,
|
||||
type OriginStats,
|
||||
} from "../../domain/analysisView/environmentManifestOrigin.js";
|
||||
import {
|
||||
mediaTypeEssence,
|
||||
readRecordedRequestLine,
|
||||
type RecordedRequestLine,
|
||||
type RequestLineExclusion,
|
||||
@@ -44,12 +48,6 @@ export interface ProbeStep {
|
||||
const BODYLESS_STATUSES: ReadonlySet<number> = new Set([204, 205, 304]);
|
||||
|
||||
const startedSchema = z.object({ startedDateTime: z.string() });
|
||||
const postDataSchema = z.object({
|
||||
request: z.object({
|
||||
postData: z.object({ mimeType: z.string(), text: z.string() }),
|
||||
}),
|
||||
});
|
||||
|
||||
const methodSchema = z.object({ request: z.object({ method: z.string() }) });
|
||||
|
||||
const startedInstant = (
|
||||
@@ -90,9 +88,11 @@ export const sendsBodies = (
|
||||
export const requestBody = (
|
||||
record: WebNetworkCaptureRecord,
|
||||
): JsonValue | undefined => {
|
||||
const parsed = postDataSchema.safeParse(record.reported);
|
||||
if (!parsed.success) return undefined;
|
||||
const { mimeType, text } = parsed.data.request.postData;
|
||||
const facts = readRecordedRequestBody(record.reported);
|
||||
if (facts.kind !== "request" || facts.postData.kind !== "object")
|
||||
return undefined;
|
||||
const { mimeType, text } = facts.postData;
|
||||
if (mimeType === undefined || text === undefined) return undefined;
|
||||
return isJsonLike(mediaTypeEssence(mimeType))
|
||||
? (parseExactJson(text) ?? text)
|
||||
: text;
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
isJsonLike,
|
||||
mediaTypeEssence,
|
||||
} from "../../domain/analysisView/recordedMediaType.js";
|
||||
|
||||
import {
|
||||
analysisViewResultSchema,
|
||||
type AnalysisViewResult,
|
||||
@@ -9,9 +14,7 @@ import {
|
||||
readRecordedResponseBody,
|
||||
type RecordedResponseBody,
|
||||
} from "../../domain/analysisView/recordedResponseBody.js";
|
||||
import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js";
|
||||
import {
|
||||
mediaTypeEssence,
|
||||
readRecordedRequestLine,
|
||||
type RecordedRequestLine,
|
||||
} from "../../domain/analysisView/webNetworkRouteTemplates.js";
|
||||
|
||||
@@ -76,6 +76,7 @@ const expectReloadsUnchanged = (books: Served): void => {
|
||||
describe("confirmed write response bodies", () => {
|
||||
it.each<{
|
||||
name: string;
|
||||
status?: number;
|
||||
content: Record<string, JsonValue>;
|
||||
shape: "recorded" | "empty";
|
||||
bodyRecorded: boolean;
|
||||
@@ -96,6 +97,23 @@ describe("confirmed write response bodies", () => {
|
||||
shape: "recorded",
|
||||
bodyRecorded: false,
|
||||
},
|
||||
{
|
||||
name: "205 without retained text",
|
||||
status: 205,
|
||||
content: { mimeType: "text/html" },
|
||||
shape: "recorded",
|
||||
bodyRecorded: false,
|
||||
},
|
||||
...[undefined, null].map((text) => ({
|
||||
name: `zero-size HTML with ${text === null ? "null" : "omitted"} text`,
|
||||
content: {
|
||||
mimeType: "text/html",
|
||||
size: 0,
|
||||
...(text === null ? { text } : {}),
|
||||
},
|
||||
shape: "recorded" as const,
|
||||
bodyRecorded: false,
|
||||
})),
|
||||
{
|
||||
name: "known empty text",
|
||||
content: { mimeType: "application/json", size: 0, text: "" },
|
||||
@@ -104,7 +122,7 @@ describe("confirmed write response bodies", () => {
|
||||
},
|
||||
])(
|
||||
"preserves $name when serving a confirmed document write",
|
||||
({ content, shape, bodyRecorded }) => {
|
||||
({ content, shape, bodyRecorded, status = 200 }) => {
|
||||
const entries: Record<string, JsonValue>[] = [
|
||||
{
|
||||
request: { method: "GET", url: "http://app.example/api/settings" },
|
||||
@@ -125,7 +143,7 @@ describe("confirmed write response bodies", () => {
|
||||
text: '{"theme":"dark"}',
|
||||
},
|
||||
},
|
||||
response: { status: 200, content },
|
||||
response: { status, content },
|
||||
},
|
||||
{
|
||||
request: { method: "GET", url: "http://app.example/api/settings" },
|
||||
@@ -169,7 +187,7 @@ describe("confirmed write response bodies", () => {
|
||||
const write = respond(patch, { body: { theme: "blue" } });
|
||||
if (!write.ok) throw write.error;
|
||||
expect(write.value).toMatchObject({
|
||||
status: 200,
|
||||
status,
|
||||
body: null,
|
||||
replay: {
|
||||
record_ordinal: 1,
|
||||
|
||||
@@ -26,7 +26,13 @@ import { webNetworkCaptureSchema } from "../../domain/webNetworkCapture.js";
|
||||
const archived = (value: boolean) => ({ ...FIRST_BOOK, archived: value });
|
||||
|
||||
describe("bodyless action evidence", () => {
|
||||
it.each([
|
||||
it.each<{
|
||||
name: string;
|
||||
postData: unknown;
|
||||
bodySize?: unknown;
|
||||
served?: boolean;
|
||||
shape?: boolean;
|
||||
}>([
|
||||
{
|
||||
name: "form",
|
||||
postData: {
|
||||
@@ -46,9 +52,45 @@ describe("bodyless action evidence", () => {
|
||||
},
|
||||
},
|
||||
{ name: "unretained body", postData: undefined },
|
||||
{
|
||||
name: "omitted body with zero size",
|
||||
postData: undefined,
|
||||
bodySize: 0,
|
||||
served: true,
|
||||
},
|
||||
{ name: "explicit null with zero size", postData: null, bodySize: 0 },
|
||||
{
|
||||
name: "params-only with zero size",
|
||||
postData: {
|
||||
mimeType: "application/json",
|
||||
params: [{ name: "archived", value: "true" }],
|
||||
},
|
||||
bodySize: 0,
|
||||
},
|
||||
{
|
||||
name: "empty text with nonempty params",
|
||||
postData: {
|
||||
mimeType: "application/json",
|
||||
text: "",
|
||||
params: [{ name: "archived", value: "true" }],
|
||||
},
|
||||
bodySize: 0,
|
||||
},
|
||||
{
|
||||
name: "explicit empty body with zero size",
|
||||
postData: { mimeType: "application/json", text: "", params: [] },
|
||||
bodySize: 0,
|
||||
served: true,
|
||||
},
|
||||
{
|
||||
name: "invalid size with retained JSON",
|
||||
postData: { mimeType: "application/json", text: '{"archived":true}' },
|
||||
bodySize: "unknown",
|
||||
shape: true,
|
||||
},
|
||||
])(
|
||||
"does not use unreadable or unretained request bodies as a repeated bodyless action: $name",
|
||||
({ postData }) => {
|
||||
"preserves literal request-body availability for repeated actions: $name",
|
||||
({ postData, bodySize = 12, served = false, shape = false }) => {
|
||||
const capture = timedCaptureEvidence([
|
||||
["GET", "/api/books", [archived(false), SECOND_BOOK]],
|
||||
["GET", "/api/books/1", archived(false)],
|
||||
@@ -60,14 +102,18 @@ describe("bodyless action evidence", () => {
|
||||
]);
|
||||
const result = capture.normalized_result as {
|
||||
records: {
|
||||
reported: { request: { postData?: unknown; bodySize?: number } };
|
||||
reported: { request: { postData?: unknown; bodySize?: unknown } };
|
||||
}[];
|
||||
};
|
||||
const request = result.records[5]!.reported.request;
|
||||
request.bodySize = 12;
|
||||
request.bodySize = bodySize;
|
||||
if (postData === undefined) delete request.postData;
|
||||
else request.postData = postData;
|
||||
const { state } = environmentStateServeInputs(
|
||||
if (shape) {
|
||||
result.records[3]!.reported.request.bodySize = bodySize;
|
||||
result.records[3]!.reported.request.postData = postData;
|
||||
}
|
||||
const { source, state } = environmentStateServeInputs(
|
||||
environmentStateServeFixture(
|
||||
analysisViewCaptureEvidence(
|
||||
webNetworkCaptureSchema.parse(capture.normalized_result),
|
||||
@@ -75,8 +121,17 @@ describe("bodyless action evidence", () => {
|
||||
),
|
||||
);
|
||||
expect(state.writes.get("post_books_archive_by_id")).toMatchObject({
|
||||
served: false,
|
||||
served,
|
||||
});
|
||||
const tool = source.manifest.tools.find(
|
||||
({ name }) => name === "post_books_archive_by_id",
|
||||
);
|
||||
expect(tool).toBeDefined();
|
||||
if (shape)
|
||||
expect(tool?.body_shape).toMatchObject({
|
||||
properties: { archived: "boolean" },
|
||||
});
|
||||
else if (bodySize === 0) expect(tool?.body_shape).toBeNull();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { z } from "zod";
|
||||
import type { AnalysisError } from "../analysisErrorBase.js";
|
||||
import { jsonValueType, parseJsonObject } from "../jsonShape.js";
|
||||
import { ok, type Result } from "../result.js";
|
||||
@@ -11,7 +10,12 @@ import {
|
||||
type WebNetworkCaptureRecord,
|
||||
} from "../webNetworkCapture.js";
|
||||
import { completeWithinViewCoverage } from "./analysisViewCoverage.js";
|
||||
import { recordedResponseBodyIsEmpty } from "./recordedResponseBody.js";
|
||||
import {
|
||||
recordedResponseBodyIsEmpty,
|
||||
readRecordedResponseDeclaration,
|
||||
} from "./recordedResponseBody.js";
|
||||
import { readRecordedRequestBody } from "./recordedExchangeBody.js";
|
||||
import { isJson, isJsonLike, mediaTypeEssence } from "./recordedMediaType.js";
|
||||
import type { UnsignedAnalysisView } from "./analysisView.js";
|
||||
import type { ScenarioResponseBodyRetention } from "./scenarioCaptureRecords.js";
|
||||
import { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js";
|
||||
@@ -41,7 +45,6 @@ import {
|
||||
GROUP_BYTES,
|
||||
RECORD_SCRATCH_BYTES,
|
||||
SEGMENT_BYTES,
|
||||
mediaTypeEssence,
|
||||
readQueryParameterNames,
|
||||
readRecordedRequestLine,
|
||||
routePartCount,
|
||||
@@ -97,24 +100,15 @@ interface GroupState extends ToolGroup {
|
||||
bodiless: boolean;
|
||||
}
|
||||
|
||||
const responseTextSchema = z.object({
|
||||
response: z.object({
|
||||
status: z.unknown().optional(),
|
||||
content: z
|
||||
.object({ text: z.unknown().optional(), size: z.unknown().optional() })
|
||||
.nullable()
|
||||
.optional(),
|
||||
}),
|
||||
});
|
||||
/** Whether a record's response had no body: a 204 or 205, an empty text, or a recorded size of zero; a missing text alone may be a body not kept. */
|
||||
const emptyResponse = (record: WebNetworkCaptureRecord): boolean => {
|
||||
const parsed = responseTextSchema.safeParse(record.reported);
|
||||
if (!parsed.success) return false;
|
||||
const { status, content } = parsed.data.response;
|
||||
/** Manifest eligibility also accepts declared bodyless statuses and a recorded zero size when no text was retained. */
|
||||
const manifestResponseIsEmpty = (record: WebNetworkCaptureRecord): boolean => {
|
||||
const response = readRecordedResponseDeclaration(record);
|
||||
if (response === undefined) return false;
|
||||
const { status, content } = response;
|
||||
return (
|
||||
status === 204 ||
|
||||
status === 205 ||
|
||||
recordedResponseBodyIsEmpty(record) ||
|
||||
recordedResponseBodyIsEmpty(record, response) ||
|
||||
((content?.text === undefined || content.text === null) &&
|
||||
content?.size === 0)
|
||||
);
|
||||
@@ -123,18 +117,6 @@ const emptyResponse = (record: WebNetworkCaptureRecord): boolean => {
|
||||
const answeredEmpty = (group: GroupState): boolean =>
|
||||
group.bodiless &&
|
||||
[...group.statuses].some((status) => status >= 200 && status < 300);
|
||||
const postDataSchema = z.object({
|
||||
request: z.object({
|
||||
postData: z
|
||||
.object({
|
||||
mimeType: z.unknown().optional(),
|
||||
text: z.unknown().optional(),
|
||||
})
|
||||
.nullable()
|
||||
.optional(),
|
||||
}),
|
||||
});
|
||||
const JSON_LIKE_TEXT = new Set(["text/plain", "text/markdown"]);
|
||||
const ASSET_ESSENCES = new Set([
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
@@ -170,17 +152,6 @@ const UNKNOWNS: Manifest["unknowns"] = [
|
||||
},
|
||||
];
|
||||
|
||||
/** `application/json` or `application/*+json`; a `+json` suffix on another type is not JSON here. */
|
||||
const isJson = (essence: string | null): boolean =>
|
||||
essence === "application/json" ||
|
||||
(essence !== null &&
|
||||
essence.startsWith("application/") &&
|
||||
essence.endsWith("+json"));
|
||||
|
||||
/** JSON, `+json`, `text/plain` or `text/markdown`: the content types a replay parses as JSON. */
|
||||
export const isJsonLike = (essence: string | null): boolean =>
|
||||
isJson(essence) || (essence !== null && JSON_LIKE_TEXT.has(essence));
|
||||
|
||||
const isApiPath = (path: string): boolean =>
|
||||
path.startsWith("/api/") || path.startsWith("/apis/");
|
||||
|
||||
@@ -260,9 +231,11 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
this.scratch,
|
||||
);
|
||||
if (line === null) return this.overBudget();
|
||||
const releaseGrouping = this.temporaryCheckpoint();
|
||||
if (line.kind === "excluded") {
|
||||
this.#unrepresented += 1;
|
||||
return ok(undefined);
|
||||
releaseGrouping();
|
||||
return this.onRecordedRequestLine(record, line);
|
||||
}
|
||||
// Coverage uses the same admitted request line as grouping, including origin and raw path.
|
||||
if (scenarioResponseBody !== undefined) {
|
||||
@@ -300,28 +273,39 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
? this.#bodyShape(record.reported)
|
||||
: ok(null);
|
||||
if (!bodyShape.ok) return bodyShape;
|
||||
return this.#append(
|
||||
const grouped = this.#append(
|
||||
record.ordinal,
|
||||
line,
|
||||
route,
|
||||
queryNames,
|
||||
{ key, group },
|
||||
{ shape: bodyShape.value, empty: emptyResponse(record) },
|
||||
{ shape: bodyShape.value, empty: manifestResponseIsEmpty(record) },
|
||||
);
|
||||
if (!grouped.ok) return grouped;
|
||||
// Keep the parsed line's admission while releasing grouping-only scratch.
|
||||
releaseGrouping();
|
||||
return this.onRecordedRequestLine(record, line);
|
||||
} finally {
|
||||
this.releaseTemporary();
|
||||
}
|
||||
}
|
||||
|
||||
/** Consume the owner's parsed request line after successful grouping, including exclusions. */
|
||||
protected onRecordedRequestLine(
|
||||
_record: WebNetworkCaptureRecord,
|
||||
_line: RecordedRequestLine,
|
||||
): Result<void, AnalysisError> {
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
/** Read the top-level key types of the group's first recorded JSON-typed request body; nested values are never walked. */
|
||||
#bodyShape(reported: unknown): Result<BodyShape | null, AnalysisError> {
|
||||
const parsed = postDataSchema.safeParse(reported);
|
||||
const body = parsed.success ? parsed.data.request.postData : undefined;
|
||||
const request = readRecordedRequestBody(reported);
|
||||
const body = request.kind === "request" ? request.postData : undefined;
|
||||
if (
|
||||
body === undefined ||
|
||||
body === null ||
|
||||
typeof body.text !== "string" ||
|
||||
typeof body.mimeType !== "string"
|
||||
body?.kind !== "object" ||
|
||||
body.text === undefined ||
|
||||
body.mimeType === undefined
|
||||
)
|
||||
return ok(null);
|
||||
// The media type is scanned and lowercased before it is known to be JSON; charge it first.
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { z } from "zod";
|
||||
import type { ScenarioResponseBodyRetention } from "./scenarioCaptureRecords.js";
|
||||
import type { AnalysisError } from "../analysisErrorBase.js";
|
||||
import { parseExactJson } from "../exactJsonNumber.js";
|
||||
import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js";
|
||||
@@ -14,10 +13,9 @@ import {
|
||||
type WebNetworkCaptureRecord,
|
||||
} from "../webNetworkCapture.js";
|
||||
import type { UnsignedAnalysisView } from "./analysisView.js";
|
||||
import {
|
||||
EnvironmentManifestAccumulator,
|
||||
isJsonLike,
|
||||
} from "./EnvironmentManifestAccumulator.js";
|
||||
import { EnvironmentManifestAccumulator } from "./EnvironmentManifestAccumulator.js";
|
||||
import { isJsonLike, mediaTypeEssence } from "./recordedMediaType.js";
|
||||
import { readRecordedRequestBody } from "./recordedExchangeBody.js";
|
||||
import { EnvironmentStateExtraction } from "./EnvironmentStateExtraction.js";
|
||||
import type { TemplateRule } from "./environmentManifestRules.js";
|
||||
import {
|
||||
@@ -32,14 +30,14 @@ import {
|
||||
import {
|
||||
base64PrefixChars,
|
||||
readRecordedResponseBody,
|
||||
readRecordedResponseDeclaration,
|
||||
recordedResponseBodyIsEmpty,
|
||||
type RecordedResponseBody,
|
||||
} from "./recordedResponseBody.js";
|
||||
import {
|
||||
ENTRY_BYTES,
|
||||
RECORD_SCRATCH_BYTES,
|
||||
mediaTypeEssence,
|
||||
readRecordedRequestLine,
|
||||
type RecordedRequestLine,
|
||||
textBytes,
|
||||
} from "./webNetworkRouteTemplates.js";
|
||||
|
||||
@@ -96,22 +94,6 @@ const redactedWriteSchema = z.object({
|
||||
request: z.object({ method: z.string() }),
|
||||
response: z.object({ status: z.number() }).optional(),
|
||||
});
|
||||
const contentSchema = z.object({
|
||||
response: z.object({ content: z.object({ text: z.string() }) }),
|
||||
});
|
||||
const requestSchema = z.object({
|
||||
request: z.object({
|
||||
postData: z.unknown().optional(),
|
||||
bodySize: z.number().optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
const requestContentSchema = z.object({
|
||||
mimeType: z.string(),
|
||||
text: z.string(),
|
||||
params: z.array(z.unknown()).optional(),
|
||||
});
|
||||
|
||||
interface RecordEntry {
|
||||
readonly facts: StateRecordFacts;
|
||||
/** Null for a write whose URL was redacted: its target is unknown, so it bounds every type. */
|
||||
@@ -157,91 +139,83 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
return false;
|
||||
}
|
||||
|
||||
override add(
|
||||
protected override onRecordedRequestLine(
|
||||
record: WebNetworkCaptureRecord,
|
||||
scenarioResponseBody?: ScenarioResponseBodyRetention,
|
||||
line: RecordedRequestLine,
|
||||
): Result<void, AnalysisError> {
|
||||
const grouped = super.add(record, scenarioResponseBody);
|
||||
if (!grouped.ok) return grouped;
|
||||
try {
|
||||
const line = readRecordedRequestLine(record.reported, true, this.scratch);
|
||||
if (line === null) return this.overBudget();
|
||||
const timing = timingSchema.safeParse(record.reported);
|
||||
// Admit normalization and fraction parsing before materializing their strings.
|
||||
if (
|
||||
timing.success &&
|
||||
!this.scratch(
|
||||
RECORD_SCRATCH_BYTES +
|
||||
textBytes(timing.data.startedDateTime.length + 6) * 3,
|
||||
)
|
||||
const timing = timingSchema.safeParse(record.reported);
|
||||
// Admit normalization and fraction parsing before materializing their strings.
|
||||
if (
|
||||
timing.success &&
|
||||
!this.scratch(
|
||||
RECORD_SCRATCH_BYTES +
|
||||
textBytes(timing.data.startedDateTime.length + 6) * 3,
|
||||
)
|
||||
return this.overBudget();
|
||||
const startInstant = timing.success
|
||||
? readRecordedDateTime(timing.data.startedDateTime)?.instant
|
||||
: undefined;
|
||||
// Keep the established millisecond interval policy independently of sorting.
|
||||
const start =
|
||||
startInstant === undefined
|
||||
? record.ordinal
|
||||
: startInstant.epochSecond * 1000 +
|
||||
Number(startInstant.fraction.slice(0, 3).padEnd(3, "0"));
|
||||
// A fraction substring may retain its normalized timestamp's backing store,
|
||||
// including trailing zeroes removed from the exact comparison value.
|
||||
const instantBytes =
|
||||
timing.success && startInstant !== undefined
|
||||
? ENTRY_BYTES + textBytes(timing.data.startedDateTime.length + 6)
|
||||
: 0;
|
||||
if (line.kind === "excluded")
|
||||
return line.reason === "url-redacted"
|
||||
? this.#redactedWrite(
|
||||
record,
|
||||
timing.success && startInstant !== undefined
|
||||
? {
|
||||
start,
|
||||
end: start + timing.data.time,
|
||||
startInstant,
|
||||
instantBytes,
|
||||
}
|
||||
: undefined,
|
||||
)
|
||||
: ok(undefined);
|
||||
// The entry keeps the line's strings and a reference to the already admitted record.
|
||||
if (
|
||||
!this.retain(
|
||||
ENTRY_BYTES * 8 +
|
||||
instantBytes +
|
||||
textBytes(
|
||||
line.method.length +
|
||||
line.origin.length +
|
||||
line.path.length +
|
||||
(line.query?.length ?? 0) +
|
||||
line.rawMime.length,
|
||||
),
|
||||
)
|
||||
)
|
||||
return this.overBudget();
|
||||
this.#entries.set(record.ordinal, {
|
||||
facts: {
|
||||
ordinal: record.ordinal,
|
||||
method: line.method,
|
||||
path: line.path,
|
||||
query: line.query === "" ? undefined : line.query,
|
||||
status: line.status,
|
||||
start,
|
||||
end:
|
||||
)
|
||||
return this.overBudget();
|
||||
const startInstant = timing.success
|
||||
? readRecordedDateTime(timing.data.startedDateTime)?.instant
|
||||
: undefined;
|
||||
// Keep the established millisecond interval policy independently of sorting.
|
||||
const start =
|
||||
startInstant === undefined
|
||||
? record.ordinal
|
||||
: startInstant.epochSecond * 1000 +
|
||||
Number(startInstant.fraction.slice(0, 3).padEnd(3, "0"));
|
||||
// A fraction substring may retain its normalized timestamp's backing store,
|
||||
// including trailing zeroes removed from the exact comparison value.
|
||||
const instantBytes =
|
||||
timing.success && startInstant !== undefined
|
||||
? ENTRY_BYTES + textBytes(timing.data.startedDateTime.length + 6)
|
||||
: 0;
|
||||
if (line.kind === "excluded")
|
||||
return line.reason === "url-redacted"
|
||||
? this.#redactedWrite(
|
||||
record,
|
||||
timing.success && startInstant !== undefined
|
||||
? start + timing.data.time
|
||||
: record.ordinal,
|
||||
},
|
||||
origin: line.origin,
|
||||
contentType: line.rawMime,
|
||||
startInstant,
|
||||
record,
|
||||
});
|
||||
return ok(undefined);
|
||||
} finally {
|
||||
this.releaseTemporary();
|
||||
}
|
||||
? {
|
||||
start,
|
||||
end: start + timing.data.time,
|
||||
startInstant,
|
||||
instantBytes,
|
||||
}
|
||||
: undefined,
|
||||
)
|
||||
: ok(undefined);
|
||||
// The entry keeps the line's strings and a reference to the already admitted record.
|
||||
if (
|
||||
!this.retain(
|
||||
ENTRY_BYTES * 8 +
|
||||
instantBytes +
|
||||
textBytes(
|
||||
line.method.length +
|
||||
line.origin.length +
|
||||
line.path.length +
|
||||
(line.query?.length ?? 0) +
|
||||
line.rawMime.length,
|
||||
),
|
||||
)
|
||||
)
|
||||
return this.overBudget();
|
||||
this.#entries.set(record.ordinal, {
|
||||
facts: {
|
||||
ordinal: record.ordinal,
|
||||
method: line.method,
|
||||
path: line.path,
|
||||
query: line.query === "" ? undefined : line.query,
|
||||
status: line.status,
|
||||
start,
|
||||
end:
|
||||
timing.success && startInstant !== undefined
|
||||
? start + timing.data.time
|
||||
: record.ordinal,
|
||||
},
|
||||
origin: line.origin,
|
||||
contentType: line.rawMime,
|
||||
startInstant,
|
||||
record,
|
||||
});
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
/** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */
|
||||
@@ -294,17 +268,22 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
// A body that is not JSON-like is refused before its decoding is charged.
|
||||
if (entry === undefined || !isJsonLike(mediaTypeEssence(entry.contentType)))
|
||||
return { kind: "unusable" };
|
||||
const content = contentSchema.safeParse(entry.record.reported);
|
||||
if (!content.success) return { kind: "unusable" };
|
||||
const declaration = readRecordedResponseDeclaration(entry.record);
|
||||
const content = declaration?.content;
|
||||
if (typeof content?.text !== "string") return { kind: "unusable" };
|
||||
// Larger bodies are never parsed; within the limit, decoding copies at most the retained text and the tree is linear in it.
|
||||
const length = content.data.response.content.text.length;
|
||||
const length = content.text.length;
|
||||
if (length > base64PrefixChars(STATE_BODY_LIMIT_BYTES))
|
||||
return { kind: "unusable" };
|
||||
if (!this.temporary(RECORD_SCRATCH_BYTES + textBytes(length) * 4))
|
||||
return null;
|
||||
const value = stateBodyJson(
|
||||
entry.contentType,
|
||||
readRecordedResponseBody(entry.record, STATE_BODY_LIMIT_BYTES),
|
||||
readRecordedResponseBody(
|
||||
entry.record,
|
||||
STATE_BODY_LIMIT_BYTES,
|
||||
declaration,
|
||||
),
|
||||
);
|
||||
return value === undefined ? { kind: "unusable" } : { kind: "json", value };
|
||||
}
|
||||
@@ -315,16 +294,30 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
use: (body: ParsedRequestBody) => T,
|
||||
): T | null {
|
||||
const entry = this.#entries.get(ordinal);
|
||||
const request = requestSchema.safeParse(entry?.record.reported);
|
||||
if (entry === undefined || !request.success)
|
||||
const request = readRecordedRequestBody(entry?.record.reported);
|
||||
if (
|
||||
entry === undefined ||
|
||||
request.kind !== "request" ||
|
||||
request.bodySize.kind === "invalid"
|
||||
)
|
||||
return use({ kind: "unusable" });
|
||||
const { postData, bodySize } = request.data.request;
|
||||
if (postData === undefined)
|
||||
return use({ kind: bodySize === 0 ? "absent" : "unusable" });
|
||||
const parsed = requestContentSchema.safeParse(postData);
|
||||
if (!parsed.success) return use({ kind: "unusable" });
|
||||
const { mimeType, text, params } = parsed.data;
|
||||
if (bodySize === 0 && text === "" && (params?.length ?? 0) === 0)
|
||||
const { postData, bodySize } = request;
|
||||
const zeroSize = bodySize.kind === "number" && bodySize.value === 0;
|
||||
if (postData.kind === "missing")
|
||||
return use({ kind: zeroSize ? "absent" : "unusable" });
|
||||
if (
|
||||
postData.kind !== "object" ||
|
||||
postData.text === undefined ||
|
||||
postData.mimeType === undefined ||
|
||||
postData.params.kind === "invalid"
|
||||
)
|
||||
return use({ kind: "unusable" });
|
||||
const { mimeType, text, params } = postData;
|
||||
if (
|
||||
zeroSize &&
|
||||
text === "" &&
|
||||
(params.kind === "array" ? params.count : 0) === 0
|
||||
)
|
||||
return use({ kind: "absent" });
|
||||
if (
|
||||
!isJsonLike(mediaTypeEssence(mimeType)) ||
|
||||
@@ -344,7 +337,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a record's response carried no body: a 204, or known decoded empty content. */
|
||||
/** State inference accepts a 204 or known decoded empty content, independently of manifest eligibility. */
|
||||
#emptyResponse(ordinal: number): boolean {
|
||||
const entry = this.#entries.get(ordinal);
|
||||
if (entry === undefined) return false;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { mediaTypeEssence } from "./recordedMediaType.js";
|
||||
import type { AnalysisError } from "../analysisErrorBase.js";
|
||||
import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js";
|
||||
import { ok, type Result } from "../result.js";
|
||||
@@ -14,7 +15,6 @@ import {
|
||||
GROUP_BYTES,
|
||||
RECORD_SCRATCH_BYTES,
|
||||
SEGMENT_BYTES,
|
||||
mediaTypeEssence,
|
||||
readQueryParameterNames,
|
||||
readRecordedRequestLine,
|
||||
scanRouteTemplate,
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/** Parsed declarations borrow source strings; they do not decode bodies or normalize media types. */
|
||||
export type RecordedRequestBody =
|
||||
| { readonly kind: "request-unavailable" }
|
||||
| {
|
||||
readonly kind: "request";
|
||||
readonly bodySize:
|
||||
| { readonly kind: "missing" | "invalid" }
|
||||
| { readonly kind: "number"; readonly value: number };
|
||||
readonly postData:
|
||||
| { readonly kind: "missing" | "null" | "malformed" }
|
||||
| {
|
||||
readonly kind: "object";
|
||||
readonly text: string | undefined;
|
||||
readonly mimeType: string | undefined;
|
||||
readonly params:
|
||||
| { readonly kind: "missing" | "invalid" }
|
||||
| { readonly kind: "array"; readonly count: number };
|
||||
};
|
||||
};
|
||||
|
||||
const object = (value: unknown): Record<string, unknown> | undefined =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: undefined;
|
||||
|
||||
/** Read HAR request body declarations once without turning omission or null into absence. */
|
||||
export const readRecordedRequestBody = (
|
||||
reported: unknown,
|
||||
): RecordedRequestBody => {
|
||||
const request = object(object(reported)?.request);
|
||||
if (request === undefined) return { kind: "request-unavailable" };
|
||||
const size = request.bodySize;
|
||||
const bodySize: Extract<
|
||||
RecordedRequestBody,
|
||||
{ kind: "request" }
|
||||
>["bodySize"] =
|
||||
size === undefined
|
||||
? { kind: "missing" }
|
||||
: typeof size === "number" && Number.isFinite(size)
|
||||
? { kind: "number", value: size }
|
||||
: { kind: "invalid" };
|
||||
const source = request.postData;
|
||||
const postData = object(source);
|
||||
return {
|
||||
kind: "request",
|
||||
bodySize,
|
||||
postData:
|
||||
postData === undefined
|
||||
? {
|
||||
kind:
|
||||
source === undefined
|
||||
? "missing"
|
||||
: source === null
|
||||
? "null"
|
||||
: "malformed",
|
||||
}
|
||||
: {
|
||||
kind: "object",
|
||||
text: typeof postData.text === "string" ? postData.text : undefined,
|
||||
mimeType:
|
||||
typeof postData.mimeType === "string"
|
||||
? postData.mimeType
|
||||
: undefined,
|
||||
params:
|
||||
postData.params === undefined
|
||||
? { kind: "missing" }
|
||||
: Array.isArray(postData.params)
|
||||
? { kind: "array", count: postData.params.length }
|
||||
: { kind: "invalid" },
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
const MEDIA_TYPE_ESSENCE =
|
||||
/^[A-Za-z0-9!#$%&'*+.^_`|~-]+\/[A-Za-z0-9!#$%&'*+.^_`|~-]+$/u;
|
||||
|
||||
const JSON_LIKE_TEXT = new Set(["text/plain", "text/markdown"]);
|
||||
|
||||
/** Lowercase type/subtype without parameters; malformed essences are unknown. */
|
||||
export const mediaTypeEssence = (reported: string): string | null => {
|
||||
const separator = reported.indexOf(";");
|
||||
let start = 0;
|
||||
let end = separator < 0 ? reported.length : separator;
|
||||
// HTTP OWS is SP/HTAB. Scan from both ends so malformed internal runs of
|
||||
// whitespace cannot cause repeated trailing-regex searches.
|
||||
while (start < end && (reported[start] === " " || reported[start] === "\t"))
|
||||
start += 1;
|
||||
while (
|
||||
end > start &&
|
||||
(reported[end - 1] === " " || reported[end - 1] === "\t")
|
||||
)
|
||||
end -= 1;
|
||||
const essence = reported.slice(start, end);
|
||||
// Validate ASCII tokens before case folding can turn Unicode into ASCII.
|
||||
return MEDIA_TYPE_ESSENCE.test(essence) ? essence.toLowerCase() : null;
|
||||
};
|
||||
|
||||
/** `application/json` or `application/*+json`; a `+json` suffix on another type is not JSON here. */
|
||||
export const isJson = (essence: string | null): boolean =>
|
||||
essence === "application/json" ||
|
||||
(essence !== null &&
|
||||
essence.startsWith("application/") &&
|
||||
essence.endsWith("+json"));
|
||||
|
||||
/** JSON, `+json`, `text/plain` or `text/markdown`: the content types a replay parses as JSON. */
|
||||
export const isJsonLike = (essence: string | null): boolean =>
|
||||
isJson(essence) || (essence !== null && JSON_LIKE_TEXT.has(essence));
|
||||
@@ -11,15 +11,32 @@ export interface RecordedResponseBody {
|
||||
readonly bodyCut?: true;
|
||||
}
|
||||
|
||||
const contentTextSchema = z.object({
|
||||
const responseDeclarationSchema = z.object({
|
||||
response: z.object({
|
||||
content: z.object({
|
||||
text: z.unknown().optional(),
|
||||
encoding: z.unknown().optional(),
|
||||
}),
|
||||
status: z.unknown().optional(),
|
||||
content: z
|
||||
.object({
|
||||
text: z.unknown().optional(),
|
||||
encoding: z.unknown().optional(),
|
||||
size: z.unknown().optional(),
|
||||
})
|
||||
.nullable()
|
||||
.optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
/** Retained response declarations shared by decoding and consumer-specific empty-body policies. */
|
||||
export const readRecordedResponseDeclaration = (
|
||||
record: WebNetworkCaptureRecord,
|
||||
) => {
|
||||
const parsed = responseDeclarationSchema.safeParse(record.reported);
|
||||
return parsed.success ? parsed.data.response : undefined;
|
||||
};
|
||||
|
||||
export type RecordedResponseDeclaration = ReturnType<
|
||||
typeof readRecordedResponseDeclaration
|
||||
>;
|
||||
|
||||
/** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */
|
||||
const strictUtf8 = (): TextDecoder =>
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
@@ -53,10 +70,13 @@ export const decodeRetainedBase64 = (
|
||||
export const readRecordedResponseBody = (
|
||||
record: WebNetworkCaptureRecord,
|
||||
limitBytes: number,
|
||||
declaration: RecordedResponseDeclaration = readRecordedResponseDeclaration(
|
||||
record,
|
||||
),
|
||||
): RecordedResponseBody => {
|
||||
const content = contentTextSchema.safeParse(record.reported);
|
||||
if (!content.success) return { bodyText: null };
|
||||
const { text, encoding } = content.data.response.content;
|
||||
const content = declaration?.content;
|
||||
if (content == null) return { bodyText: null };
|
||||
const { text, encoding } = content;
|
||||
if (typeof text !== "string") return { bodyText: null };
|
||||
if (encoding === undefined) {
|
||||
// A hidden content property could have been the encoding declaration;
|
||||
@@ -80,8 +100,11 @@ export const readRecordedResponseBody = (
|
||||
/** Prove decoded emptiness with a bounded prefix, preserving encoding and cut uncertainty. */
|
||||
export const recordedResponseBodyIsEmpty = (
|
||||
record: WebNetworkCaptureRecord,
|
||||
declaration: RecordedResponseDeclaration = readRecordedResponseDeclaration(
|
||||
record,
|
||||
),
|
||||
): boolean => {
|
||||
const body = readRecordedResponseBody(record, 0);
|
||||
const body = readRecordedResponseBody(record, 0, declaration);
|
||||
return (
|
||||
body.bodyText === "" &&
|
||||
body.bodyEncoding === undefined &&
|
||||
|
||||
@@ -52,9 +52,6 @@ const responseSchema = z.object({
|
||||
}),
|
||||
});
|
||||
const contentSchema = z.object({ mimeType: z.unknown().optional() });
|
||||
const MEDIA_TYPE_ESSENCE =
|
||||
/^[A-Za-z0-9!#$%&'*+.^_`|~-]+\/[A-Za-z0-9!#$%&'*+.^_`|~-]+$/u;
|
||||
|
||||
// Conservative representation allowances, not an enforced V8 heap limit.
|
||||
// Fixed charges include the accumulator, bucket/set entries, array growth and
|
||||
// page/schema projection. Text includes UTF-16 storage and two worst-case JSON
|
||||
@@ -67,25 +64,6 @@ export const CAPTURE_CONTAINER_BYTES = 512;
|
||||
const SEGMENT_SCRATCH_BYTES = 512;
|
||||
export const textBytes = (length: number): number => 32 + length * 14;
|
||||
|
||||
/** Lowercase type/subtype without parameters; malformed essences are unknown. */
|
||||
export const mediaTypeEssence = (reported: string): string | null => {
|
||||
const separator = reported.indexOf(";");
|
||||
let start = 0;
|
||||
let end = separator < 0 ? reported.length : separator;
|
||||
// HTTP OWS is SP/HTAB. Scan from both ends so malformed internal runs of
|
||||
// whitespace cannot cause repeated trailing-regex searches.
|
||||
while (start < end && (reported[start] === " " || reported[start] === "\t"))
|
||||
start += 1;
|
||||
while (
|
||||
end > start &&
|
||||
(reported[end - 1] === " " || reported[end - 1] === "\t")
|
||||
)
|
||||
end -= 1;
|
||||
const essence = reported.slice(start, end);
|
||||
// Validate ASCII tokens before case folding can turn Unicode into ASCII.
|
||||
return MEDIA_TYPE_ESSENCE.test(essence) ? essence.toLowerCase() : null;
|
||||
};
|
||||
|
||||
/** Route parts of a template: one more than its separators. */
|
||||
export const routePartCount = (template: string): number => {
|
||||
let count = 1;
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js";
|
||||
import {
|
||||
ArtifactResourceRetryFailure,
|
||||
type ArtifactResourceScope,
|
||||
} from "../../artifacts/ArtifactResourceScope.js";
|
||||
import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js";
|
||||
import { mediaTypeEssence } from "../../domain/analysisView/webNetworkRouteTemplates.js";
|
||||
import { isExactJsonNumber } from "../../domain/exactJsonNumber.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { safeParseJson } from "../../domain/safeJson.js";
|
||||
|
||||
Reference in new issue
Block a user