chore: merge main into codex/audit-python-bytecode

src/application/apiContract/ApiContractService.test.ts

@@ -5,8 +5,6 @@ import { expect, it, vi } from "vitest";
 import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
 import { apiContractSchema } from "../../domain/apiContract/apiContract.js";
 import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
-import { parseEvidence } from "../../domain/evidence.js";
-import { projectAnalysisView } from "../../domain/analysisView/analysisView.js";
 import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
 import { inspectApiContractEvidence } from "./ApiContractService.js";
 import * as readOwner from "./ApiContractRead.js";
@@ -319,89 +317,3 @@ it("cancels a cooperative document traversal after the actual verified file read
     await resources.close();
   }
 });
-
-it.each(["between-paths", "after-projection"] as const)(
-  "preserves authenticated OpenAPI facts on cancellation %s",
-  async (stage) => {
-    const bytes = JSON.stringify({
-      openapi: "3.1.2",
-      info: { title: "Cancellation", version: "1" },
-      paths: { "/first": { get: {} }, "/second": { post: {} } },
-    });
-    const { path } = await inputFile(bytes);
-    const resources = new ArtifactResourceScope();
-    const controller = new AbortController();
-    const realParse = parser.parseApiContractSteps;
-    try {
-      const baseline = await inspectApiContractEvidence({ path }, resources);
-      if (!baseline.ok) throw baseline.error;
-      const complete = apiContractSchema.parse(
-        baseline.value.normalized_result,
-      );
-      const scheduled = vi
-        .spyOn(parser, "parseApiContractSteps")
-        .mockImplementationOnce(function* (...args) {
-          const steps = realParse(...args);
-          if (stage === "between-paths") {
-            expect(steps.next().done).toBe(false);
-            expect(steps.next().done).toBe(false);
-            setImmediate(() => controller.abort(stage));
-            return yield* steps;
-          }
-          const result = yield* steps;
-          setImmediate(() => controller.abort(stage));
-          return result;
-        });
-      try {
-        const result = await inspectApiContractEvidence({ path }, resources, {
-          signal: controller.signal,
-        });
-        if (result.ok) throw new Error("Expected cancellation");
-        expect(result.error._tag).toBe("AnalysisCancelledError");
-        expect(result.error.cause).toBe(stage);
-        const evidence = parseEvidence(result.error.partialObservation);
-        const retained = apiContractSchema.parse(evidence.normalized_result);
-        expect(evidence.subject).toEqual(baseline.value.subject);
-        expect(evidence.parameters).toEqual(baseline.value.parameters);
-        expect(retained.artifact).toEqual(complete.artifact);
-        expect(retained.contract_id).toBe(complete.contract_id);
-        if (stage === "between-paths") {
-          expect(retained.operations).toEqual(complete.operations.slice(0, 1));
-          expect(retained.source_coverage.status).toBe("partial");
-          expect(evidence.evidence_id).not.toBe(baseline.value.evidence_id);
-          const page = projectAnalysisView(
-            {
-              evidenceId: evidence.evidence_id,
-              operation: evidence.operation,
-              predicateType: evidence.predicate_type,
-              normalizedResult: evidence.normalized_result,
-              limitations: evidence.limitations,
-            },
-            {
-              kind: "api-contract",
-              collection: "operations",
-              offset: 0,
-              limit: 1,
-            },
-          );
-          expect(page).toMatchObject({
-            ok: true,
-            value: {
-              parent_evidence_id: evidence.evidence_id,
-              items: [{ path: "/first", state: "declared" }],
-              coverage: { exhausted: true },
-              source_coverage: { status: "partial" },
-            },
-          });
-        } else {
-          expect(retained).toEqual(complete);
-          expect(evidence.evidence_id).toBe(baseline.value.evidence_id);
-        }
-      } finally {
-        scheduled.mockRestore();
-      }
-    } finally {
-      await resources.close();
-    }
-  },
-);

src/application/apiContract/ApiContractService.ts

@@ -11,8 +11,6 @@ import { err, ok, type Result } from "../../domain/result.js";
 import {
   API_CONTRACT_OPERATION as operation,
   API_CONTRACT_PROVIDER,
-  apiContractSchema,
-  type ApiContract,
   type ApiContractArtifact,
 } from "../../domain/apiContract/apiContract.js";
 import {
@@ -72,32 +70,8 @@ function* documentSteps(document: unknown, artifact: ApiContractArtifact) {
   return yield* parseApiContractSteps(document, artifact);
 }
 
-type OpenApiInput = Extract<
-  InspectApiContractInput,
-  { format: "openapi-json" }
->;
-
-const documentEvidence = (input: OpenApiInput, result: ApiContract): Evidence =>
-  createEvidence(
-    { path: input.path, format: "file", sha256: result.artifact.sha256 },
-    API_CONTRACT_PROVIDER,
-    {
-      operation,
-      predicateType: "rea.api-contract-inventory",
-      parameters: {
-        path: input.path,
-        format: input.format,
-        max_file_bytes: input.max_file_bytes,
-      },
-      result,
-      confidence: "observed",
-      limitations: result.limitations,
-      locations: [{ kind: "artifact-path", path: input.path }],
-    },
-  );
-
 const importDocument = async (
-  input: OpenApiInput,
+  input: Extract<InspectApiContractInput, { format: "openapi-json" }>,
   bytes: Buffer,
   sha256: string,
   signal?: AbortSignal,
@@ -112,16 +86,26 @@ const importDocument = async (
     }),
     signal,
   );
-  if (!parsed.ok) {
-    const partial = apiContractSchema.safeParse(
-      parsed.error.partialObservation,
-    );
-    if (partial.success)
-      parsed.error.partialObservation = documentEvidence(input, partial.data);
-    return parsed;
-  }
+  if (!parsed.ok) return parsed;
+  const result = parsed.value;
   return completeProviderEvidence(
-    documentEvidence(input, parsed.value),
+    createEvidence(
+      { path: input.path, format: "file", sha256 },
+      API_CONTRACT_PROVIDER,
+      {
+        operation,
+        predicateType: "rea.api-contract-inventory",
+        parameters: {
+          path: input.path,
+          format: input.format,
+          max_file_bytes: input.max_file_bytes,
+        },
+        result,
+        confidence: "observed",
+        limitations: result.limitations,
+        locations: [{ kind: "artifact-path", path: input.path }],
+      },
+    ),
     signal,
   );
 };

src/artifacts/pyinstaller/MarshalSubset.test.ts

@@ -20,6 +20,11 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
   ]);
 });
 
+it("reads marshal one-byte string tags as Latin-1", () => {
+  for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
+    expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
+});
+
 it("decodes CPython marshal Unicode with surrogatepass", () => {
   expect(
     readMarshalSubset(

src/artifacts/pyinstaller/MarshalSubset.ts

@@ -168,14 +168,8 @@ const readAsciiBytes = (
 ): Result<string, PyInstallerFailure> => {
   const bytes = readBytes(cursor, length);
   if (!bytes.ok) return bytes;
-  if (bytes.value.some((byte) => byte > 0x7f))
-    return err(
-      pyInstallerFailure(
-        "format",
-        "PYZ marshal ASCII string contains a non-ASCII byte",
-      ),
-    );
-  return ok(bytes.value.toString("ascii"));
+  // These tags use CPython's one-byte Unicode constructor, not an ASCII check.
+  return ok(bytes.value.toString("latin1"));
 };
 
 const readUtf8 = (

src/domain/analysisErrorBase.ts

@@ -1,7 +1,6 @@
 import type { PartialTaskMeasurement } from "./desktop/taskDifficulty.js";
 import type { AppleIpsPartialObservation } from "./native/appleIps.js";
 import type { PlaywrightTrace } from "./playwrightTrace/playwrightTrace.js";
-import type { ApiContract } from "./apiContract/apiContract.js";
 import type { GraphqlContract } from "./apiContract/graphqlContract.js";
 import type { SavedAnalysisSnapshotPartialObservation } from "./savedAnalysisSnapshot.js";
 import type { AnalysisSettingLimit } from "./analysisSettings.js";
@@ -499,7 +498,6 @@ export type AnalysisPartialObservation =
     }
   | { readonly kind: "unretained-evidence"; readonly evidence: Evidence }
   | { readonly kind: "retained-evidence"; readonly evidence_id: string }
-  | ApiContract
   | GraphqlContract
   | PlaywrightTrace
   | SavedAnalysisSnapshotPartialObservation

src/domain/analysisView/apiContractView.ts

@@ -185,7 +185,7 @@ export const projectApiContractView = (
     unknowns:
       context.source_coverage.status === "partial"
         ? [
-            "This inventory is partial. Inspect limitations for coverage and query path-item-references and diagnostics for any retained source pointers.",
+            "Unresolved or malformed declarations limit this inventory; query path-item-references and diagnostics for exact source pointers.",
           ]
         : [],
   };

src/domain/apiContract/parseApiContract.test.ts

@@ -3,7 +3,6 @@ import { readFileSync } from "node:fs";
 import { expect, it } from "vitest";
 import { apiContractSchema, type ApiContractArtifact } from "./apiContract.js";
 import { parseApiContractSteps } from "./parseApiContract.js";
-import { AnalysisCancelledError } from "../analysisErrorCore.js";
 
 const artifact: ApiContractArtifact = {
   path: "/contract.json",
@@ -306,39 +305,9 @@ it("handles 3.2 QUERY and case-sensitive additional methods, diagnosing invalid
 
 it("bounds expanded identities and source pointers before building a giant row", () => {
   const result = parse(
-    document({
-      "/retained": { get: {} },
-      [`/${"a".repeat(1_100_000)}`]: { get: {} },
-    }),
+    document({ [`/${"a".repeat(1_100_000)}`]: { get: {} } }),
   );
   expect(result.ok).toBe(false);
-  if (result.ok) throw new Error("Expected a retained-facts refusal");
-  expect(result.error._tag).toBe("AnalysisResourceConstraintError");
-  const partial = apiContractSchema.parse(result.error.partialObservation);
-  expect(partial.operations.map((row) => row.path)).toEqual(["/retained"]);
-  expect(partial.source_coverage.status).toBe("partial");
-});
-
-it("retains an interpreted path when cancellation interrupts traversal", () => {
-  const input = document({
-    "/first": { get: { operationId: "first" } },
-    "/second": { get: { operationId: "second" } },
-  });
-  const steps = parseApiContractSteps(input, artifact);
-  for (let checkpoint = 0; checkpoint < 3; checkpoint++)
-    expect(steps.next().done).toBe(false);
-  const cancellation = new AnalysisCancelledError("inspect_api_contract");
-  const stopped = steps.throw(cancellation);
-  if (!stopped.done || stopped.value.ok)
-    throw new Error("Expected cancellation");
-  expect(stopped.value.error).toBe(cancellation);
-  const partial = apiContractSchema.parse(cancellation.partialObservation);
-  const complete = report(input);
-  expect(partial.operations).toEqual(complete.operations.slice(0, 1));
-  expect(partial.summary.operation_slots).toBe(1);
-  expect(partial.source_coverage).toMatchObject({
-    status: "partial",
-    path_entries_examined: 1,
-  });
-  expect(complete.source_coverage.status).toBe("complete-within-scope");
+  if (!result.ok)
+    expect(result.error._tag).toBe("AnalysisResourceConstraintError");
 });

src/domain/apiContract/parseApiContract.ts

@@ -1,5 +1,4 @@
 import {
-  AnalysisCancelledError,
   AnalysisInputError,
   AnalysisResourceConstraintError,
   AnalysisUnsupportedTargetError,
@@ -317,17 +316,14 @@ function* pathItem(
   return ok(undefined);
 }
 
-interface ContractReportContext {
-  readonly openapi: string;
-  readonly info: ApiContract["info"];
-  readonly pathsState: ApiContract["source_coverage"]["paths_state"];
-  examined: number;
-}
-
-const contractReport = (
+const completedInventory = (
   inventory: ContractInventory,
-  context: ContractReportContext,
-  incomplete = false,
+  context: {
+    openapi: string;
+    info: ApiContract["info"];
+    pathsState: ApiContract["source_coverage"]["paths_state"];
+    examined: number;
+  },
 ): ApiContract => {
   const artifact = inventory.artifact;
   let declared = 0;
@@ -352,9 +348,7 @@ const contractReport = (
     source_coverage: {
       scope: "inline-path-operations",
       status:
-        incomplete ||
-        inventory.references.length > 0 ||
-        inventory.diagnostics.length > 0
+        inventory.references.length > 0 || inventory.diagnostics.length > 0
           ? "partial"
           : "complete-within-scope",
       paths_state: context.pathsState,
@@ -364,11 +358,6 @@ const contractReport = (
       duplicate_json_members: "last-value-wins",
     },
     limitations: [
-      ...(incomplete
-        ? [
-            "Traversal stopped before the inventory was complete. Counts describe the retained rows.",
-          ]
-        : []),
       "Operations are document declarations, not evidence of runtime implementation, reachability or authorization.",
       "Only inline operations under paths are inventoried; callbacks, webhooks, components, servers, parameters, schemas, request/response contracts and security are not analyzed.",
       "No reference is resolved, including local fragments. A Path Item reference makes its effective operations unknown; adjacent inline slots are preserved without merging referenced definitions.",
@@ -379,32 +368,6 @@ const contractReport = (
   };
 };
 
-function* collectPaths(
-  sourcePaths: Record<string, unknown>,
-  inventory: ContractInventory,
-  context: ContractReportContext,
-  modern: boolean,
-): Generator<void, Result<void, AnalysisError>> {
-  for (const path in sourcePaths) {
-    if (!Object.hasOwn(sourcePaths, path) || path.startsWith("x-")) continue;
-    yield;
-    context.examined++;
-    // Reserve expanded pointer/path strings before escaping or building a row.
-    const admitted = inventory.reserve(512 + path.length * 16);
-    if (!admitted.ok) return admitted;
-    const pointer = `/paths/${pointerPart(path)}`;
-    const read = path.startsWith("/")
-      ? yield* pathItem(sourcePaths[path], { inventory, path, pointer, modern })
-      : inventory.diagnostic(
-          "invalid-path-key",
-          pointer,
-          "A Paths entry must start with /; it was not interpreted as an endpoint.",
-        );
-    if (!read.ok) return read;
-  }
-  return ok(undefined);
-}
-
 /** Enumerate declarations only; never resolve references, servers, callbacks or runtime expressions. */
 export function* parseApiContractSteps(
   value: unknown,
@@ -454,32 +417,36 @@ export function* parseApiContractSteps(
     );
     if (!diagnosed.ok) return diagnosed;
   }
-  const context: ContractReportContext = {
-    openapi: root.openapi,
-    info: info.value,
-    pathsState,
-    examined: 0,
-  };
-  const interrupted = (error: AnalysisError) => {
-    const retainedRows =
-      inventory.operations.length +
-      inventory.references.length +
-      inventory.diagnostics.length;
-    if (retainedRows > 0)
-      error.retainPartialObservation(contractReport(inventory, context, true));
-    return err(error);
-  };
-  try {
-    const read = yield* collectPaths(
-      paths ?? {},
-      inventory,
-      context,
-      version[1] === "2",
-    );
-    if (!read.ok) return interrupted(read.error);
-  } catch (cause: unknown) {
-    if (cause instanceof AnalysisCancelledError) return interrupted(cause);
-    throw cause;
+  let examined = 0;
+  const sourcePaths = paths ?? {};
+  for (const path in sourcePaths) {
+    if (!Object.hasOwn(sourcePaths, path) || path.startsWith("x-")) continue;
+    yield;
+    examined++;
+    // Reserve expanded pointer/path strings before escaping or building a row.
+    const admitted = inventory.reserve(512 + path.length * 16);
+    if (!admitted.ok) return admitted;
+    const pointer = `/paths/${pointerPart(path)}`;
+    const read = path.startsWith("/")
+      ? yield* pathItem(sourcePaths[path], {
+          inventory,
+          path,
+          pointer,
+          modern: version[1] === "2",
+        })
+      : inventory.diagnostic(
+          "invalid-path-key",
+          pointer,
+          "A Paths entry must start with /; it was not interpreted as an endpoint.",
+        );
+    if (!read.ok) return read;
   }
-  return ok(contractReport(inventory, context));
+  return ok(
+    completedInventory(inventory, {
+      openapi: root.openapi,
+      info: info.value,
+      pathsState,
+      examined,
+    }),
+  );
 }

src/python/PythonPycParser.test.ts

@@ -407,6 +407,30 @@ it("charges long digits to the string budget and keeps large previews bounded",
   );
 });
 
+it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
+  for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
+    const encoded = Buffer.concat([
+      Buffer.from([tag]),
+      tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
+      Buffer.from([0xe9]),
+    ]);
+    const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
+    const set = Buffer.concat([
+      Buffer.from([0x3c]),
+      u32(2),
+      encoded,
+      sameUnicode,
+    ]);
+    const report = root(parse(withConsts([encoded, set])));
+    expect(report.consts[0]).toMatchObject({
+      kind: "str",
+      value: "é",
+      bytes: 1,
+    });
+    expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
+  }
+});
+
 it("decodes marshal strings with CPython surrogatepass rules", () => {
   const report = parse(
     withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
@@ -1534,6 +1558,36 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
   expect(pythonPycSchema.safeParse(report).success).toBe(true);
 });
 
+it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
+  const links = 3000;
+  const codes = Array.from({ length: links }, (_, index) =>
+    marshalCode("py311", {
+      flagged: true,
+      name: "f",
+      consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
+    }),
+  );
+  const bytes = Buffer.concat([
+    pythonPycHeader(3531),
+    marshalCode("py311", {
+      constsObject: Buffer.concat([
+        Buffer.from([0x28]),
+        u32(links + 1),
+        ...codes,
+        Buffer.from([0x3c]),
+        u32(1),
+        marshalRef(links - 1),
+      ]),
+    }),
+  ]);
+  const report = parseWith(bytes, { max_objects: 65_536 });
+  expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
+  expect(report.coverage).toMatchObject({
+    status: "complete",
+    code_objects: links + 1,
+  });
+});
+
 it("rejects code CPython treats as corrupt instead of complete coverage", () => {
   const file = (
     layout: "py37" | "py38" | "py311",

src/python/PythonPycParser.ts

@@ -332,7 +332,7 @@ class MarshalReader {
   readonly #codeKeys = new WeakMap<CodeNode, string>();
   readonly #codeIdentities = new WeakMap<CodeNode, number>();
   readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
-  /** Python hashability per code object. False while that check is in progress. */
+  /** Completed Python hashability checks per code object. */
   readonly #codeHashable = new WeakMap<CodeNode, boolean>();
   /** Tuples whose items are still being read; CPython cannot hash them yet. */
   readonly #readingTuples = new WeakSet<MarshalValue>();
@@ -436,13 +436,19 @@ class MarshalReader {
         };
       case TYPE_INTERNED:
       case TYPE_UNICODE:
-        return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
+        return this.string(
+          this.nonNegativeInt32("Marshal string length"),
+          "utf8",
+        );
       case TYPE_ASCII:
       case TYPE_ASCII_INTERNED:
-        return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
+        return this.string(
+          this.nonNegativeInt32("Marshal string length"),
+          "latin1",
+        );
       case TYPE_SHORT_ASCII:
       case TYPE_SHORT_ASCII_INTERNED:
-        return this.utf8(this.u8(), true);
+        return this.string(this.u8(), "latin1");
       case TYPE_TUPLE:
         return this.sequenceValue(
           "tuple",
@@ -1309,13 +1315,70 @@ class MarshalReader {
   #isCodeHashable(node: CodeNode): boolean {
     const known = this.#codeHashable.get(node);
     if (known !== undefined) return known;
-    this.#codeHashable.set(node, false);
-    const material =
-      this.#codeEquality.get(node) ??
-      malformed("Code object is missing equality material.");
-    const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
-    this.#codeHashable.set(node, hashable);
-    return hashable;
+    const frames: { value: MarshalValue; finish: boolean }[] = [
+      { value: { kind: "code", value: node }, finish: false },
+    ];
+    const gray = new Set<MarshalValue | CodeNode>();
+    const complete = new Set<MarshalValue>();
+    const reject = (): false => {
+      this.#codeHashable.set(node, false);
+      return false;
+    };
+    while (frames.length > 0) {
+      this.#takeKeyStep();
+      const frame = frames.pop()!;
+      const value = frame.value;
+      const identity = value.kind === "code" ? value.value : value;
+      if (frame.finish) {
+        gray.delete(identity);
+        if (value.kind === "code") {
+          const material =
+            this.#codeEquality.get(value.value) ??
+            malformed("Code object is missing equality material.");
+          // Child code checks are now complete, so the existing tuple key
+          // walker preserves its equality and expansion budgets without recursion.
+          const hashable =
+            this.#equalityKey(material.constsTuple, "hash") !== null;
+          this.#codeHashable.set(value.value, hashable);
+          if (!hashable) return reject();
+        } else complete.add(value);
+        continue;
+      }
+      if (value.kind === "code") {
+        const cached = this.#codeHashable.get(value.value);
+        if (cached === false) return reject();
+        if (cached === true) continue;
+      } else {
+        const cached = this.#keyCache.get(value);
+        if (cached === null) return reject();
+        if (cached !== undefined || complete.has(value)) continue;
+      }
+      if (
+        gray.has(identity) ||
+        this.#readingTuples.has(value) ||
+        value.kind === "list" ||
+        value.kind === "set" ||
+        value.kind === "dict" ||
+        value.kind === "null"
+      )
+        return reject();
+      const children =
+        value.kind === "code"
+          ? [
+              (
+                this.#codeEquality.get(value.value) ??
+                malformed("Code object is missing equality material.")
+              ).constsTuple,
+            ]
+          : value.kind === "tuple" || value.kind === "frozenset"
+            ? value.items
+            : [];
+      gray.add(identity);
+      frames.push({ value, finish: true });
+      for (let index = children.length - 1; index >= 0; index -= 1)
+        frames.push({ value: children[index]!, finish: false });
+    }
+    return true;
   }
 
   #codeIdentity(node: CodeNode): number {
@@ -1492,17 +1555,20 @@ class MarshalReader {
     );
   }
 
-  utf8(length: number, ascii: boolean): MarshalValue {
+  string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
     const bytes = this.blob(length);
-    if (ascii) {
-      if (bytes.some((byte) => byte > 0x7f))
-        malformed("Marshal ASCII string contains a non-ASCII byte.");
+    if (encoding === "latin1") {
+      // CPython reads the ASCII-tagged forms as one-byte Unicode, even when
+      // high bytes appear. Identity must agree with the UTF-8 string form.
+      const value = bytes.toString("latin1");
       return {
         kind: "str",
-        value: bytes.toString("latin1"),
+        value,
         previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
         bytes: length,
-        raw: bytes,
+        raw: bytes.some((byte) => byte > 0x7f)
+          ? Buffer.from(value, "utf8")
+          : bytes,
       };
     }
     const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);