Compare commits

..
Author SHA1 Message Date
morluto 2eb48add14 fix(python): validate shared pyc header facts 2026-11-10 21:41:05 -05:00
12 changed files with 118 additions and 287 deletions

No files matched your search

@@ -66,6 +66,28 @@ const report: PythonPyc = {
],
};
it.each([
{
flags: 1,
hash_based: true,
source_hash_hex: "",
source_mtime: null,
source_size: null,
},
{ flags: 1 },
])("refuses inconsistent provider header facts %j", async (header) => {
const result = await new PythonPycService({
close: async () => {},
identity,
inspect: async () =>
ok({ ...report, header: { ...report.header, ...header } }),
}).inspect({ path: report.artifact.path });
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisOutputError" },
});
});
it("rejects a relative path and extra arguments before reading", async () => {
let calls = 0;
const service = new PythonPycService({
@@ -559,7 +559,6 @@ export const erlangBeamSummaryViewResultSchema = z.strictObject({
});
export const erlangBeamTableViewResultSchema = z.strictObject({
...viewResultBase,
coverage: nullableCollectionViewCoverageSchema,
kind: z.literal("beam"),
parent_operation: z.literal("inspect_erlang_beam"),
view: erlangBeamViewRequestSchema,
@@ -625,7 +624,6 @@ export const rustBinarySummaryViewResultSchema = z.strictObject({
export const rustBinaryPageViewResultSchema = z.strictObject({
...viewResultBase,
coverage: nullableCollectionViewCoverageSchema,
...rustBinaryViewContextSchema.shape,
kind: z.literal("rust"),
parent_operation: z.literal("inspect_rust_binary"),
@@ -1507,7 +1505,6 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [
z.strictObject({
...viewResultBase,
...goBinaryViewContextSchema.shape,
coverage: nullableCollectionViewCoverageSchema,
kind: z.literal("go"),
parent_operation: z.literal("inspect_go_binary"),
view: goBinaryViewRequestSchema,
@@ -1,125 +0,0 @@
import { expect, it } from "vitest";
import { createEvidence } from "../evidence.js";
import type { ErlangBeam } from "../erlang/beam.js";
import type { ModelArtifact } from "../model/modelArtifact.js";
import {
projectAnalysisView,
type AnalysisViewParent,
} from "./analysisView.js";
const parent = (
operation: string,
result: ErlangBeam | ModelArtifact,
): AnalysisViewParent => {
const evidence = createEvidence(
{ path: result.artifact.path, sha256: "a".repeat(64), format: "file" },
{ id: "fixture", name: "Fixture reader", version: "1" },
{ operation, parameters: {}, result },
);
return {
predicateType: evidence.predicate_type,
evidenceId: evidence.evidence_id,
operation,
subject: evidence.subject,
normalizedResult: result,
limitations: [],
};
};
const beam: ErlangBeam = {
artifact: {
path: "/artifacts/module.beam",
sha256: "a".repeat(64),
bytes: 12,
},
form: { location: { offset: 0, bytes: 12 }, size: 4 },
chunks: [],
atoms: null,
atom_encoding: null,
exports: null,
imports: null,
lambdas: null,
code: null,
module: null,
coverage: { status: "complete", examined_chunks: 0, atoms: 0 },
limitations: [],
};
it.each(["atoms", "exports", "imports", "lambdas"] as const)(
"distinguishes unavailable BEAM %s from a decoded empty table",
(collection) => {
for (const available of [false, true]) {
const result = projectAnalysisView(
parent("inspect_erlang_beam", {
...beam,
[collection]: available ? [] : null,
}),
{ kind: "beam", collection, offset: 7, limit: 1 },
);
if (!result.ok) throw result.error;
expect(result.value.coverage).toEqual({
status: available ? "empty" : "unavailable",
examined: 0,
total: available ? 0 : null,
next_offset: null,
exhausted: available,
});
expect(result.value).toMatchObject({ page: { collection, items: [] } });
if (!available)
expect(result.value.unknowns.join(" ")).toContain(
`No ${collection} table was decoded`,
);
}
},
);
const model: ModelArtifact = {
artifact: {
path: "/artifacts/model.gguf",
file_size: 32,
header_sha256: "a".repeat(64),
header_length: 32,
sha256: "a".repeat(64),
bytes_read: 32,
read_scope: "full-file",
},
format: "gguf",
version: 3,
byte_order: "little",
alignment: 32,
tensor_data_offset: 32,
metadata: [],
tensors: [],
totals: {
confidence: "derived",
parameter_count: "0",
tensor_bytes: 0,
by_type: [],
},
coverage: {
header: "complete",
tensor_data: "not-interpreted",
array_values: "preview",
tensor_ranges: "partial",
storage_sizes: "partial",
},
limitations: [
"Array metadata retains only the requested preview; storage sizes remain unknown.",
],
};
it.each(["metadata", "tensors", "types"] as const)(
"retains producer limitations on model %s pages with no envelope warnings",
(collection) => {
const input = parent("inspect_model_artifact", model);
const result = projectAnalysisView(input, {
kind: "model-artifact",
collection,
offset: 0,
limit: 1,
});
if (!result.ok) throw result.error;
expect(result.value.limitations).toEqual(model.limitations);
expect(input.limitations).toEqual([]);
},
);
+3 -13
View File
@@ -13,7 +13,6 @@ import {
boundedViewMessages,
completeWithinViewCoverage,
pageViewCoverage,
unavailableViewCoverage,
} from "./analysisViewCoverage.js";
import { selectViewRows } from "./selectViewRows.js";
@@ -121,7 +120,7 @@ type Row = NonNullable<ErlangBeam[Table]>[number];
type BeamView = z.infer<typeof erlangBeamViewRequestSchema>;
/** Select already parsed rows; original atom/lambda indexes are never repurposed. */
const selectRows = (source: readonly Row[], view: BeamView) => {
const selectRows = (rows: readonly Row[] | null, view: BeamView) => {
const names = view.names === undefined ? undefined : new Set(view.names);
const indices =
view.indices === undefined ? undefined : new Set(view.indices);
@@ -139,6 +138,7 @@ const selectRows = (source: readonly Row[], view: BeamView) => {
),
source_index,
});
const source = rows ?? [];
if (names === undefined && indices === undefined && modules === undefined)
return {
rows: source
@@ -223,17 +223,7 @@ export const projectErlangBeamView = (
summary,
coverage: completeWithinViewCoverage(chunks.length, chunks.length),
});
const rows = parsed.value[view.collection];
if (rows === null)
return ok({
...shared,
kind: "beam" as const,
view,
beam: summary,
page: { collection: view.collection, items: [] },
coverage: unavailableViewCoverage(),
});
const selected = selectRows(rows, view);
const selected = selectRows(parsed.value[view.collection], view);
return ok({
...shared,
kind: "beam" as const,
@@ -238,53 +238,3 @@ it("refuses string_literals view when unrequested and projects filtered/truncate
],
});
});
it.each(["dependencies", "settings"] as const)(
"distinguishes unavailable Go %s from decoded empty records",
(collection) => {
for (const available of [false, true]) {
const input = parent();
const result = projectAnalysisView(
{
...input,
normalizedResult: {
...report,
build_info: available
? {
header_offset: 0,
encoding: "inline",
go_version: null,
go_version_bytes_base64: "",
module_text: null,
module_bytes_base64: "",
version_location: { offset: 32, bytes: 0 },
module_location: { offset: 32, bytes: 0 },
module: {
path: null,
main: null,
dependencies: [],
settings: [],
unparsed_lines: [],
unparsed_line_bytes_base64: [],
complete: true,
},
}
: null,
},
},
{ kind: "go", collection, offset: 3, limit: 1 },
);
if (!result.ok) throw result.error;
expect(result.value.coverage).toEqual({
status: available ? "empty" : "unavailable",
examined: 0,
total: available ? 0 : null,
next_offset: null,
exhausted: available,
});
expect(result.value).toMatchObject({
source_coverage: { build_info_available: available },
});
}
},
);
+2 -11
View File
@@ -20,7 +20,6 @@ import {
boundedViewMessages,
completeWithinViewCoverage,
pageViewCoverage,
unavailableViewCoverage,
} from "./analysisViewCoverage.js";
import { analysisViewInputError } from "./analysisViewErrors.js";
import { createAnalysisViewResultReader } from "./analysisViewParent.js";
@@ -323,18 +322,10 @@ export const projectGoBinaryView = (
),
});
}
if (info === null)
return ok({
kind: "go" as const,
view,
...shared,
page: { collection: view.collection, rows: [] },
coverage: unavailableViewCoverage(),
});
if (view.collection === "dependencies") {
const paths = view.paths === undefined ? undefined : new Set(view.paths);
const selected = selectViewRows(
info.module.dependencies,
info?.module.dependencies ?? [],
view,
(row, index) =>
(indices?.has(index) ?? true) && (paths?.has(row.path) ?? true),
@@ -362,7 +353,7 @@ export const projectGoBinaryView = (
}
const keys = view.keys === undefined ? undefined : new Set(view.keys);
const selected = selectViewRows(
info.module.settings,
info?.module.settings ?? [],
view,
(row, index) =>
(indices?.has(index) ?? true) && (keys?.has(row.key) ?? true),
+1 -1
View File
@@ -217,7 +217,7 @@ export const projectModelArtifactView = (
parent_digest: parent.evidenceId.slice(3),
artifact: { path: report.artifact.path, sha256: report.artifact.sha256 },
limitations: boundedViewMessages(
[...parent.limitations, ...report.limitations],
parent.limitations,
parent.evidenceId,
"limitations",
),
@@ -206,40 +206,3 @@ it("pages matching original indices and omits dependency vectors unless selected
next_offset: null,
});
});
it.each(["packages", "edges"] as const)(
"distinguishes unavailable Rust %s from decoded empty records",
(collection) => {
for (const available of [false, true]) {
const metadata = report.auditable_metadata!;
const result = projectAnalysisView(
parent({
...report,
auditable_metadata: available
? {
...metadata,
packages: [],
root_package_index: null,
dependency_edges: [],
cyclic_dependency_indices: [],
}
: null,
}),
{ kind: "rust", collection, offset: 3, limit: 1 },
);
if (!result.ok) throw result.error;
expect(result.value.coverage).toEqual({
status: available ? "empty" : "unavailable",
examined: 0,
total: available ? 0 : null,
next_offset: null,
exhausted: available,
});
expect(result.value).toMatchObject({
source_coverage: {
embedded_records: available ? "complete" : "unknown",
},
});
}
},
);
+2 -12
View File
@@ -22,7 +22,6 @@ import {
boundedViewMessages,
completeWithinViewCoverage,
pageViewCoverage,
unavailableViewCoverage,
} from "./analysisViewCoverage.js";
import { analysisViewInputError } from "./analysisViewErrors.js";
import { selectViewRows } from "./selectViewRows.js";
@@ -191,15 +190,6 @@ export const projectRustBinaryView = (
summary,
coverage: completeWithinViewCoverage(1, 1),
});
if (metadata === null)
return ok({
kind: "rust" as const,
view,
...shared,
rust: summary,
page: { collection: view.collection, rows: [] },
coverage: unavailableViewCoverage(),
});
const indices =
view.indices === undefined ? undefined : new Set(view.indices);
const fields =
@@ -209,7 +199,7 @@ export const projectRustBinaryView = (
const versions =
view.versions === undefined ? undefined : new Set(view.versions);
const selected = selectViewRows(
metadata.packages,
metadata?.packages ?? [],
view,
(row, index) =>
(indices?.has(index) ?? true) &&
@@ -248,7 +238,7 @@ export const projectRustBinaryView = (
const to =
view.to_indices === undefined ? undefined : new Set(view.to_indices);
const selected = selectViewRows(
metadata.dependency_edges,
metadata?.dependency_edges ?? [],
view,
(row, index) =>
(indices?.has(index) ?? true) &&
+40
View File
@@ -3,9 +3,49 @@ import {
inspectPythonPycInputSchema,
pythonPycMarshalProfile,
PYTHON_PYC_MARSHAL_PROFILES,
pythonPycHeaderSchema,
} from "./pythonPyc.js";
import { pythonBytecodeIdentity } from "./pythonBytecodeIdentity.js";
const hashHeader = {
magic: 3531,
python_version: "3.12",
code_layout: "py311",
flags: 3,
hash_based: true,
check_source: true,
source_hash_hex: "0123456789abcdef",
source_mtime: null,
source_size: null,
};
it.each(["", "ab", "g".repeat(16)])(
"rejects an invalid source hash %j",
(source_hash_hex) => {
expect(
pythonPycHeaderSchema.safeParse({ ...hashHeader, source_hash_hex })
.success,
).toBe(false);
},
);
it.each([
{ hash_based: false },
{ check_source: false },
{ source_hash_hex: null },
{ source_mtime: 0 },
{ source_size: 0 },
])("keeps header facts consistent with the recorded flags: %j", (changed) => {
expect(
pythonPycHeaderSchema.safeParse({ ...hashHeader, ...changed }).success,
).toBe(false);
});
it("preserves unknown raw header flags and valid hash bytes", () => {
const header = { ...hashHeader, flags: 0x80000003 };
expect(pythonPycHeaderSchema.parse(header)).toEqual(header);
});
it("keeps pinned CPython identity separate from standalone marshal coverage", () => {
expect(pythonPycMarshalProfile(3394)).toEqual({
code_layout: "py37",
+47 -34
View File
@@ -205,17 +205,53 @@ const pythonCodeObjectSchema = z.strictObject({
});
export type PythonPycCodeObject = z.infer<typeof pythonCodeObjectSchema>;
export const pythonPycHeaderSchema = z.strictObject({
magic: z.number().int().min(0).max(0xffff),
python_version: z.string().min(1),
code_layout: z.enum(PYTHON_PYC_CODE_LAYOUTS),
flags: uint,
hash_based: z.boolean(),
check_source: z.boolean(),
source_hash_hex: z.string().nullable(),
source_mtime: uint.nullable(),
source_size: uint.nullable(),
});
export const pythonPycHeaderSchema = z
.strictObject({
magic: z.number().int().min(0).max(0xffff),
python_version: z.string().min(1),
code_layout: z.enum(PYTHON_PYC_CODE_LAYOUTS),
flags: uint,
hash_based: z.boolean(),
check_source: z.boolean(),
source_hash_hex: z
.string()
.regex(/^[0-9a-f]{16}$/u)
.nullable(),
source_mtime: uint.nullable(),
source_size: uint.nullable(),
})
.superRefine((header, context) => {
for (const [field, flag] of [
["hash_based", PYTHON_PYC_HASH_BASED],
["check_source", PYTHON_PYC_CHECK_SOURCE],
] as const)
if (header[field] !== ((header.flags & flag) !== 0))
context.addIssue({
code: "custom",
path: [field],
message: `${field} disagrees with its recorded header flag.`,
});
if (header.hash_based === (header.source_hash_hex === null))
context.addIssue({
code: "custom",
path: ["source_hash_hex"],
message:
"Hash-based pycs require an 8-byte source hash; others must omit it.",
});
if (header.hash_based === (header.source_mtime !== null))
context.addIssue({
code: "custom",
path: ["source_mtime"],
message: "Timestamp pycs require mtime; hash-based pycs must omit it.",
});
if (header.hash_based === (header.source_size !== null))
context.addIssue({
code: "custom",
path: ["source_size"],
message:
"Timestamp pycs require source_size; hash-based pycs must omit it.",
});
});
/** Bounded CPython 3.7–3.13 pyc observation; marshal is walked, bytecode is not decompiled. */
export const pythonPycSchema = z
@@ -253,29 +289,6 @@ export const pythonPycSchema = z
message: "Marshal byte count disagrees with the header-stripped file.",
});
}
if (report.header.hash_based === (report.header.source_hash_hex === null)) {
context.addIssue({
code: "custom",
path: ["header", "source_hash_hex"],
message:
"Hash-based pycs require an 8-byte source hash; others must omit it.",
});
}
if (report.header.hash_based === (report.header.source_mtime !== null)) {
context.addIssue({
code: "custom",
path: ["header", "source_mtime"],
message: "Timestamp pycs require mtime; hash-based pycs must omit it.",
});
}
if (report.header.hash_based === (report.header.source_size !== null)) {
context.addIssue({
code: "custom",
path: ["header", "source_size"],
message:
"Timestamp pycs require source_size; hash-based pycs must omit it.",
});
}
if (report.coverage.code_objects !== report.code_objects.length) {
context.addIssue({
code: "custom",
+1 -1
View File
@@ -368,7 +368,7 @@ mcpTest(
const page = goPage(parseEvidence(empty.structuredContent));
expect(page).toMatchObject({
page: { rows: [] },
coverage: { status: "unavailable", total: null, exhausted: false },
coverage: { total: 0 },
source_coverage: {
build_info_available: false,
module_parse_complete: null,