chore: merge main into codex/audit-python-bytecode

src/artifacts/pyinstaller/MarshalSubset.test.ts

@@ -20,6 +20,11 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
   ]);
 });
 
+it("reads marshal one-byte string tags as Latin-1", () => {
+  for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
+    expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
+});
+
 it("decodes CPython marshal Unicode with surrogatepass", () => {
   expect(
     readMarshalSubset(

src/artifacts/pyinstaller/MarshalSubset.ts

@@ -168,14 +168,8 @@ const readAsciiBytes = (
 ): Result<string, PyInstallerFailure> => {
   const bytes = readBytes(cursor, length);
   if (!bytes.ok) return bytes;
-  if (bytes.value.some((byte) => byte > 0x7f))
+  // These tags use CPython's one-byte Unicode constructor, not an ASCII check.
-    return err(
+  return ok(bytes.value.toString("latin1"));
-      pyInstallerFailure(
-        "format",
-        "PYZ marshal ASCII string contains a non-ASCII byte",
-      ),
-    );
-  return ok(bytes.value.toString("ascii"));
 };
 
 const readUtf8 = (

src/domain/reconstructionVerification.test.ts

@@ -615,108 +615,3 @@ it("keeps a valid comparison selected for the wrong claim kind as caller input",
     });
   }
 });
-
-describe("retained reconstruction source coverage", () => {
-  const omittedTerminalCapture = {
-    ...EMPTY_PROCESS_CAPTURE_EXAMPLE,
-    truncated: true,
-    truncation_details: {
-      ...EMPTY_PROCESS_CAPTURE_EXAMPLE.truncation_details,
-      raw_terminal: {
-        ...EMPTY_PROCESS_CAPTURE_EXAMPLE.truncation_details.raw_terminal,
-        observed_frames: 1,
-        observed_bytes: 1,
-      },
-    },
-  };
-
-  it("keeps old unchanged terminal and overall claims unknown while retaining unrelated claims", () => {
-    const left = source("1", "controlled-replay");
-    const right = source("2", "controlled-replay", "observed", {
-      captureResult: omittedTerminalCapture,
-    });
-    const comparison = processComparison(left, right, processResult());
-    const bundle = createEvidenceBundle([left, right, comparison]);
-    const before = JSON.stringify(bundle);
-    const result = verifyReconstruction(
-      {
-        name: "Retained capture coverage",
-        claims: ["terminal", "overall", "exit", "interaction"].map(
-          (dimension) => ({
-            kind: "behavioral",
-            claim_id: dimension,
-            title: dimension,
-            dimension,
-            comparison_evidence_id: comparison.evidence_id,
-          }),
-        ),
-      },
-      bundle,
-    );
-    expect(
-      result.claims.items.map(({ claim_id, status }) => [claim_id, status]),
-    ).toEqual([
-      ["exit", "pass"],
-      ["interaction", "pass"],
-      ["overall", "unknown"],
-      ["terminal", "unknown"],
-    ]);
-    expect(
-      result.claims.items.find(({ claim_id }) => claim_id === "terminal")
-        ?.limitations,
-    ).toContain(
-      `Source ${right.evidence_id}: terminal observation coverage is incomplete.`,
-    );
-    expect(JSON.stringify(bundle)).toBe(before);
-  });
-
-  it("retains a decisive changed claim despite omitted terminal observations", () => {
-    const left = source("1", "controlled-replay");
-    const right = source("2", "controlled-replay", "observed", {
-      captureResult: omittedTerminalCapture,
-    });
-    const comparison = processComparison(left, right, processResult("changed"));
-    const claims = ["terminal", "overall"].map((dimension) => ({
-      kind: "behavioral",
-      claim_id: dimension,
-      title: dimension,
-      dimension,
-      comparison_evidence_id: comparison.evidence_id,
-    }));
-    expect(
-      verifyReconstruction(
-        { name: "Observed disagreement", claims },
-        createEvidenceBundle([left, right, comparison]),
-      ),
-    ).toMatchObject({
-      status: "fail",
-      summary: { failed: 2, unknown: 0 },
-    });
-  });
-
-  it.each(["status", "kind"] as const)(
-    "rejects a summary inconsistent with actual claim %s",
-    (field) => {
-      const left = source("1", "controlled-replay");
-      const right = source("2", "controlled-replay");
-      const comparison = processComparison(left, right, processResult());
-      const result = verifyReconstruction(
-        behavioralSpec(comparison),
-        createEvidenceBundle([left, right, comparison]),
-      );
-      const inconsistent = {
-        ...result,
-        claims: {
-          items: result.claims.items.map((claim) =>
-            field === "status"
-              ? { ...claim, status: "fail", observed_status: "changed" }
-              : { ...claim, kind: "structural-function" },
-          ),
-        },
-      };
-      expect(
-        reconstructionVerificationResultSchema.safeParse(inconsistent).success,
-      ).toBe(false);
-    },
-  );
-});

src/domain/reconstructionVerification.ts

@@ -15,16 +15,7 @@ import { analysisInputErrorFromIssues } from "./inputIssueProjection.js";
 import { readEvidencePayload } from "./evidenceValidation.js";
 import { functionDossierSchema } from "./native/functionDossier.js";
 import { assessFunctionCallCoverage } from "./functionCallCoverage.js";
-import {
+import { processCaptureSchema } from "./process/processCapture.js";
-  processCaptureSchema,
-  type UnverifiedProcessCapture,
-} from "./process/processCapture.js";
-import { processSourceTruncated } from "./process/processCaptureCoverage.js";
-import {
-  PROCESS_COMPARISON_DIMENSIONS,
-  type ProcessComparisonDimension,
-} from "./process/processComparison.js";
-import type { ProcessObservationSource } from "./process/processObservation.js";
 import { digestCanonicalValue } from "./canonicalDigest.js";
 import {
   reconstructionClaimResultSchema,
@@ -54,14 +45,6 @@ interface FunctionSourceFacts {
   readonly callCoverageDiagnostics: readonly string[];
 }
 
-interface ValidatedSourceFacts {
-  readonly functions: readonly FunctionSourceFacts[];
-  readonly processes: readonly {
-    readonly evidence_id: string;
-    readonly capture: UnverifiedProcessCapture;
-  }[];
-}
-
 /** Verify only explicitly declared claims; no global equivalence is inferred. */
 export const verifyReconstruction = (
   specificationInput: unknown,
@@ -161,18 +144,17 @@ const evaluateClaim = (
       );
     return source;
   });
-  const sourceFacts = readValidatedSourceFacts(claim, sources);
+  const functionSourceFacts = readValidatedSourceFacts(claim, sources);
   const observation = comparisonObservation(claim, comparison);
   const observed = observation.status;
   const callCoverage = sourceCallCoverage(
     claim,
-    sourceFacts.functions,
+    functionSourceFacts,
     observation.hasIndependentFunctionChange === true,
   );
   const blockers = [
     ...authorityLimitations(claim, sources, sides),
     ...callCoverage.blockers,
-    ...sourceProcessCoverage(claim, sourceFacts.processes, observed),
   ];
   const relevantUnknowns = reconstructionClaimUnknowns(
     heads,
@@ -207,7 +189,7 @@ const evaluateClaim = (
 const readValidatedSourceFacts = (
   claim: Claim,
   sources: readonly Evidence[],
-): ValidatedSourceFacts => {
+): readonly FunctionSourceFacts[] => {
   const isExpectedType = (source: Evidence): boolean => {
     if (claim.kind === "behavioral")
       return (
@@ -234,7 +216,6 @@ const readValidatedSourceFacts = (
       },
     );
   const functionSources: FunctionSourceFacts[] = [];
-  const processSources: ValidatedSourceFacts["processes"][number][] = [];
   for (const source of sources) {
     if (claim.kind === "behavioral") {
       const capture = readEvidencePayload(
@@ -242,10 +223,6 @@ const readValidatedSourceFacts = (
         source.normalized_result,
       );
       if (!capture.ok) throw capture.error;
-      processSources.push({
-        evidence_id: source.evidence_id,
-        capture: capture.value,
-      });
     } else if (claim.kind === "structural-function") {
       const parsedDossier = readEvidencePayload(
         functionDossierSchema,
@@ -279,7 +256,7 @@ const readValidatedSourceFacts = (
       }
     }
   }
-  return { functions: functionSources, processes: processSources };
+  return functionSources;
 };
 
 const validateComparisonIdentity = (
@@ -427,44 +404,6 @@ const authorityLimitations = (
   return limitations;
 };
 
-const processObservationSourceByDimension: Readonly<
-  Record<ProcessComparisonDimension, ProcessObservationSource>
-> = {
-  terminal: "terminal_rendered",
-  interaction: "interaction",
-  exit: "lifecycle",
-  filesystem: "filesystem",
-  process: "process",
-};
-
-const sourceProcessCoverage = (
-  claim: Claim,
-  sources: ValidatedSourceFacts["processes"],
-  observed: ObservedStatus,
-): string[] => {
-  if (
-    claim.kind !== "behavioral" ||
-    ["changed", "added", "removed", "contradiction"].includes(observed)
-  )
-    return [];
-  const dimensions =
-    claim.dimension === "overall"
-      ? PROCESS_COMPARISON_DIMENSIONS
-      : [claim.dimension];
-  return sources.flatMap(({ evidence_id, capture }) =>
-    dimensions.flatMap((dimension) =>
-      processSourceTruncated(
-        capture,
-        processObservationSourceByDimension[dimension],
-      )
-        ? [
-            `Source ${evidence_id}: ${dimension} observation coverage is incomplete.`,
-          ]
-        : [],
-    ),
-  );
-};
-
 const sourceCallCoverage = (
   claim: Claim,
   functionSources: readonly FunctionSourceFacts[],

src/domain/reconstructionVerificationSchemas.ts

@@ -118,28 +118,9 @@ export const reconstructionVerificationResultSchema = z
   })
   .superRefine((result, context) => {
     const { summary } = result;
-    const claims = result.claims.items;
     const expectedStatus =
       summary.failed > 0 ? "fail" : summary.unknown > 0 ? "unknown" : "pass";
     const issues = [
-      [
-        summary.passed !==
-          claims.filter(({ status }) => status === "pass").length ||
-          summary.failed !==
-            claims.filter(({ status }) => status === "fail").length ||
-          summary.unknown !==
-            claims.filter(({ status }) => status === "unknown").length,
-        "Claim status counts must summarize retained claims",
-        ["summary"],
-      ],
-      [
-        summary.behavioral !==
-          claims.filter(({ kind }) => kind === "behavioral").length ||
-          summary.structural !==
-            claims.filter(({ kind }) => kind !== "behavioral").length,
-        "Claim kind counts must summarize retained claims",
-        ["summary"],
-      ],
       [
         summary.total !== summary.passed + summary.failed + summary.unknown,
         "Claim status counts must equal the summary total",

src/python/PythonPycParser.test.ts

@@ -407,6 +407,30 @@ it("charges long digits to the string budget and keeps large previews bounded",
   );
 });
 
+it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
+  for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
+    const encoded = Buffer.concat([
+      Buffer.from([tag]),
+      tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
+      Buffer.from([0xe9]),
+    ]);
+    const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
+    const set = Buffer.concat([
+      Buffer.from([0x3c]),
+      u32(2),
+      encoded,
+      sameUnicode,
+    ]);
+    const report = root(parse(withConsts([encoded, set])));
+    expect(report.consts[0]).toMatchObject({
+      kind: "str",
+      value: "é",
+      bytes: 1,
+    });
+    expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
+  }
+});
+
 it("decodes marshal strings with CPython surrogatepass rules", () => {
   const report = parse(
     withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
@@ -1534,6 +1558,36 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
   expect(pythonPycSchema.safeParse(report).success).toBe(true);
 });
 
+it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
+  const links = 3000;
+  const codes = Array.from({ length: links }, (_, index) =>
+    marshalCode("py311", {
+      flagged: true,
+      name: "f",
+      consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
+    }),
+  );
+  const bytes = Buffer.concat([
+    pythonPycHeader(3531),
+    marshalCode("py311", {
+      constsObject: Buffer.concat([
+        Buffer.from([0x28]),
+        u32(links + 1),
+        ...codes,
+        Buffer.from([0x3c]),
+        u32(1),
+        marshalRef(links - 1),
+      ]),
+    }),
+  ]);
+  const report = parseWith(bytes, { max_objects: 65_536 });
+  expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
+  expect(report.coverage).toMatchObject({
+    status: "complete",
+    code_objects: links + 1,
+  });
+});
+
 it("rejects code CPython treats as corrupt instead of complete coverage", () => {
   const file = (
     layout: "py37" | "py38" | "py311",

src/python/PythonPycParser.ts

@@ -332,7 +332,7 @@ class MarshalReader {
   readonly #codeKeys = new WeakMap<CodeNode, string>();
   readonly #codeIdentities = new WeakMap<CodeNode, number>();
   readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
-  /** Python hashability per code object. False while that check is in progress. */
+  /** Completed Python hashability checks per code object. */
   readonly #codeHashable = new WeakMap<CodeNode, boolean>();
   /** Tuples whose items are still being read; CPython cannot hash them yet. */
   readonly #readingTuples = new WeakSet<MarshalValue>();
@@ -436,13 +436,19 @@ class MarshalReader {
         };
       case TYPE_INTERNED:
       case TYPE_UNICODE:
-        return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
+        return this.string(
+          this.nonNegativeInt32("Marshal string length"),
+          "utf8",
+        );
       case TYPE_ASCII:
       case TYPE_ASCII_INTERNED:
-        return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
+        return this.string(
+          this.nonNegativeInt32("Marshal string length"),
+          "latin1",
+        );
       case TYPE_SHORT_ASCII:
       case TYPE_SHORT_ASCII_INTERNED:
-        return this.utf8(this.u8(), true);
+        return this.string(this.u8(), "latin1");
       case TYPE_TUPLE:
         return this.sequenceValue(
           "tuple",
@@ -1309,13 +1315,70 @@ class MarshalReader {
   #isCodeHashable(node: CodeNode): boolean {
     const known = this.#codeHashable.get(node);
     if (known !== undefined) return known;
-    this.#codeHashable.set(node, false);
+    const frames: { value: MarshalValue; finish: boolean }[] = [
-    const material =
+      { value: { kind: "code", value: node }, finish: false },
-      this.#codeEquality.get(node) ??
+    ];
-      malformed("Code object is missing equality material.");
+    const gray = new Set<MarshalValue | CodeNode>();
-    const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
+    const complete = new Set<MarshalValue>();
-    this.#codeHashable.set(node, hashable);
+    const reject = (): false => {
-    return hashable;
+      this.#codeHashable.set(node, false);
+      return false;
+    };
+    while (frames.length > 0) {
+      this.#takeKeyStep();
+      const frame = frames.pop()!;
+      const value = frame.value;
+      const identity = value.kind === "code" ? value.value : value;
+      if (frame.finish) {
+        gray.delete(identity);
+        if (value.kind === "code") {
+          const material =
+            this.#codeEquality.get(value.value) ??
+            malformed("Code object is missing equality material.");
+          // Child code checks are now complete, so the existing tuple key
+          // walker preserves its equality and expansion budgets without recursion.
+          const hashable =
+            this.#equalityKey(material.constsTuple, "hash") !== null;
+          this.#codeHashable.set(value.value, hashable);
+          if (!hashable) return reject();
+        } else complete.add(value);
+        continue;
+      }
+      if (value.kind === "code") {
+        const cached = this.#codeHashable.get(value.value);
+        if (cached === false) return reject();
+        if (cached === true) continue;
+      } else {
+        const cached = this.#keyCache.get(value);
+        if (cached === null) return reject();
+        if (cached !== undefined || complete.has(value)) continue;
+      }
+      if (
+        gray.has(identity) ||
+        this.#readingTuples.has(value) ||
+        value.kind === "list" ||
+        value.kind === "set" ||
+        value.kind === "dict" ||
+        value.kind === "null"
+      )
+        return reject();
+      const children =
+        value.kind === "code"
+          ? [
+              (
+                this.#codeEquality.get(value.value) ??
+                malformed("Code object is missing equality material.")
+              ).constsTuple,
+            ]
+          : value.kind === "tuple" || value.kind === "frozenset"
+            ? value.items
+            : [];
+      gray.add(identity);
+      frames.push({ value, finish: true });
+      for (let index = children.length - 1; index >= 0; index -= 1)
+        frames.push({ value: children[index]!, finish: false });
+    }
+    return true;
   }
 
   #codeIdentity(node: CodeNode): number {
@@ -1492,17 +1555,20 @@ class MarshalReader {
     );
   }
 
-  utf8(length: number, ascii: boolean): MarshalValue {
+  string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
     const bytes = this.blob(length);
-    if (ascii) {
+    if (encoding === "latin1") {
-      if (bytes.some((byte) => byte > 0x7f))
+      // CPython reads the ASCII-tagged forms as one-byte Unicode, even when
-        malformed("Marshal ASCII string contains a non-ASCII byte.");
+      // high bytes appear. Identity must agree with the UTF-8 string form.
+      const value = bytes.toString("latin1");
       return {
         kind: "str",
-        value: bytes.toString("latin1"),
+        value,
         previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
         bytes: length,
-        raw: bytes,
+        raw: bytes.some((byte) => byte > 0x7f)
+          ? Buffer.from(value, "utf8")
+          : bytes,
       };
     }
     const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);