chore: merge main into codex/audit-python-bytecode

.github/workflows/real-web-network-captures.yml

@@ -56,8 +56,7 @@ jobs:
         run: |
           capture_tool_root="${RUNNER_TEMP}/rea-mitmproxy-12.2.3"
           mkdir -p "${capture_tool_root}"
-          # This immutable GET may reset mid-body; retry it before checksum verification.
-          curl --fail --location --retry 3 --retry-all-errors --max-time 180 \
+          curl --fail --location --retry 3 --max-time 180 \
             https://downloads.mitmproxy.org/12.2.3/mitmproxy-12.2.3-linux-x86_64.tar.gz \
             --output "${capture_tool_root}/mitmproxy.tar.gz"
           echo '2e95286b618fa6fd33e5e62a78c2e5112571d85f42ec2bac29b97ee242bdb5c5  mitmproxy.tar.gz' \

src/artifacts/pyinstaller/MarshalSubset.test.ts

@@ -20,6 +20,11 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
   ]);
 });
 
+it("reads marshal one-byte string tags as Latin-1", () => {
+  for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
+    expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
+});
+
 it("decodes CPython marshal Unicode with surrogatepass", () => {
   expect(
     readMarshalSubset(

src/artifacts/pyinstaller/MarshalSubset.ts

@@ -168,14 +168,8 @@ const readAsciiBytes = (
 ): Result<string, PyInstallerFailure> => {
   const bytes = readBytes(cursor, length);
   if (!bytes.ok) return bytes;
-  if (bytes.value.some((byte) => byte > 0x7f))
-    return err(
-      pyInstallerFailure(
-        "format",
-        "PYZ marshal ASCII string contains a non-ASCII byte",
-      ),
-    );
-  return ok(bytes.value.toString("ascii"));
+  // These tags use CPython's one-byte Unicode constructor, not an ASCII check.
+  return ok(bytes.value.toString("latin1"));
 };
 
 const readUtf8 = (

src/python/PythonPycParser.test.ts

@@ -407,6 +407,30 @@ it("charges long digits to the string budget and keeps large previews bounded",
   );
 });
 
+it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
+  for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
+    const encoded = Buffer.concat([
+      Buffer.from([tag]),
+      tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
+      Buffer.from([0xe9]),
+    ]);
+    const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
+    const set = Buffer.concat([
+      Buffer.from([0x3c]),
+      u32(2),
+      encoded,
+      sameUnicode,
+    ]);
+    const report = root(parse(withConsts([encoded, set])));
+    expect(report.consts[0]).toMatchObject({
+      kind: "str",
+      value: "é",
+      bytes: 1,
+    });
+    expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
+  }
+});
+
 it("decodes marshal strings with CPython surrogatepass rules", () => {
   const report = parse(
     withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
@@ -1534,6 +1558,36 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
   expect(pythonPycSchema.safeParse(report).success).toBe(true);
 });
 
+it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
+  const links = 3000;
+  const codes = Array.from({ length: links }, (_, index) =>
+    marshalCode("py311", {
+      flagged: true,
+      name: "f",
+      consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
+    }),
+  );
+  const bytes = Buffer.concat([
+    pythonPycHeader(3531),
+    marshalCode("py311", {
+      constsObject: Buffer.concat([
+        Buffer.from([0x28]),
+        u32(links + 1),
+        ...codes,
+        Buffer.from([0x3c]),
+        u32(1),
+        marshalRef(links - 1),
+      ]),
+    }),
+  ]);
+  const report = parseWith(bytes, { max_objects: 65_536 });
+  expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
+  expect(report.coverage).toMatchObject({
+    status: "complete",
+    code_objects: links + 1,
+  });
+});
+
 it("rejects code CPython treats as corrupt instead of complete coverage", () => {
   const file = (
     layout: "py37" | "py38" | "py311",

src/python/PythonPycParser.ts

@@ -332,7 +332,7 @@ class MarshalReader {
   readonly #codeKeys = new WeakMap<CodeNode, string>();
   readonly #codeIdentities = new WeakMap<CodeNode, number>();
   readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
-  /** Python hashability per code object. False while that check is in progress. */
+  /** Completed Python hashability checks per code object. */
   readonly #codeHashable = new WeakMap<CodeNode, boolean>();
   /** Tuples whose items are still being read; CPython cannot hash them yet. */
   readonly #readingTuples = new WeakSet<MarshalValue>();
@@ -436,13 +436,19 @@ class MarshalReader {
         };
       case TYPE_INTERNED:
       case TYPE_UNICODE:
-        return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
+        return this.string(
+          this.nonNegativeInt32("Marshal string length"),
+          "utf8",
+        );
       case TYPE_ASCII:
       case TYPE_ASCII_INTERNED:
-        return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
+        return this.string(
+          this.nonNegativeInt32("Marshal string length"),
+          "latin1",
+        );
       case TYPE_SHORT_ASCII:
       case TYPE_SHORT_ASCII_INTERNED:
-        return this.utf8(this.u8(), true);
+        return this.string(this.u8(), "latin1");
       case TYPE_TUPLE:
         return this.sequenceValue(
           "tuple",
@@ -1309,13 +1315,70 @@ class MarshalReader {
   #isCodeHashable(node: CodeNode): boolean {
     const known = this.#codeHashable.get(node);
     if (known !== undefined) return known;
-    this.#codeHashable.set(node, false);
-    const material =
-      this.#codeEquality.get(node) ??
-      malformed("Code object is missing equality material.");
-    const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
-    this.#codeHashable.set(node, hashable);
-    return hashable;
+    const frames: { value: MarshalValue; finish: boolean }[] = [
+      { value: { kind: "code", value: node }, finish: false },
+    ];
+    const gray = new Set<MarshalValue | CodeNode>();
+    const complete = new Set<MarshalValue>();
+    const reject = (): false => {
+      this.#codeHashable.set(node, false);
+      return false;
+    };
+    while (frames.length > 0) {
+      this.#takeKeyStep();
+      const frame = frames.pop()!;
+      const value = frame.value;
+      const identity = value.kind === "code" ? value.value : value;
+      if (frame.finish) {
+        gray.delete(identity);
+        if (value.kind === "code") {
+          const material =
+            this.#codeEquality.get(value.value) ??
+            malformed("Code object is missing equality material.");
+          // Child code checks are now complete, so the existing tuple key
+          // walker preserves its equality and expansion budgets without recursion.
+          const hashable =
+            this.#equalityKey(material.constsTuple, "hash") !== null;
+          this.#codeHashable.set(value.value, hashable);
+          if (!hashable) return reject();
+        } else complete.add(value);
+        continue;
+      }
+      if (value.kind === "code") {
+        const cached = this.#codeHashable.get(value.value);
+        if (cached === false) return reject();
+        if (cached === true) continue;
+      } else {
+        const cached = this.#keyCache.get(value);
+        if (cached === null) return reject();
+        if (cached !== undefined || complete.has(value)) continue;
+      }
+      if (
+        gray.has(identity) ||
+        this.#readingTuples.has(value) ||
+        value.kind === "list" ||
+        value.kind === "set" ||
+        value.kind === "dict" ||
+        value.kind === "null"
+      )
+        return reject();
+      const children =
+        value.kind === "code"
+          ? [
+              (
+                this.#codeEquality.get(value.value) ??
+                malformed("Code object is missing equality material.")
+              ).constsTuple,
+            ]
+          : value.kind === "tuple" || value.kind === "frozenset"
+            ? value.items
+            : [];
+      gray.add(identity);
+      frames.push({ value, finish: true });
+      for (let index = children.length - 1; index >= 0; index -= 1)
+        frames.push({ value: children[index]!, finish: false });
+    }
+    return true;
   }
 
   #codeIdentity(node: CodeNode): number {
@@ -1492,17 +1555,20 @@ class MarshalReader {
     );
   }
 
-  utf8(length: number, ascii: boolean): MarshalValue {
+  string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
     const bytes = this.blob(length);
-    if (ascii) {
-      if (bytes.some((byte) => byte > 0x7f))
-        malformed("Marshal ASCII string contains a non-ASCII byte.");
+    if (encoding === "latin1") {
+      // CPython reads the ASCII-tagged forms as one-byte Unicode, even when
+      // high bytes appear. Identity must agree with the UTF-8 string form.
+      const value = bytes.toString("latin1");
       return {
         kind: "str",
-        value: bytes.toString("latin1"),
+        value,
         previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
         bytes: length,
-        raw: bytes,
+        raw: bytes.some((byte) => byte > 0x7f)
+          ? Buffer.from(value, "utf8")
+          : bytes,
       };
     }
     const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);

tests/boundary/filesystem/nativeProfileDownload.test.ts

@@ -1,176 +0,0 @@
-import { createHash } from "node:crypto";
-import { readFile, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { setTimeout as delay } from "node:timers/promises";
-import { fileURLToPath } from "node:url";
-
-import { describe, expect, it, onTestFinished } from "vitest";
-import { parse } from "yaml";
-import { z } from "zod";
-
-import { createVerifierHttpServer } from "../../../scripts/lib/verifier-http-server.mjs";
-import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
-import { executeWorkflowFixture } from "../../support/workflowGit.js";
-
-const upstream =
-  "https://downloads.mitmproxy.org/12.2.3/mitmproxy-12.2.3-linux-x86_64.tar.gz";
-const digest =
-  "2e95286b618fa6fd33e5e62a78c2e5112571d85f42ec2bac29b97ee242bdb5c5";
-
-async function fixture(
-  outcome: "recover" | "reset" | "not-found" | "bad-digest",
-) {
-  const directory = await createTestTempDirectory("rea-native-download-");
-  const archivePath = join(directory, "fixture.tar.gz");
-  await writeFile(join(directory, "mitmdump"), "fixture native profile\n");
-  await executeWorkflowFixture("tar", [
-    "-czf",
-    archivePath,
-    "-C",
-    directory,
-    "mitmdump",
-  ]);
-  const archive = await readFile(archivePath);
-  let requests = 0;
-  const { server, close } = createVerifierHttpServer(
-    async (request, response) => {
-      requests++;
-      if (outcome === "not-found") {
-        response.writeHead(404).end();
-        return;
-      }
-      response.writeHead(200, { "Content-Length": archive.length });
-      if (outcome === "reset" || (outcome === "recover" && requests === 1)) {
-        response.write(archive.subarray(0, Math.floor(archive.length / 2)));
-        // Deliver a partial body, then a real TCP reset (curl 56), not an HTTP error.
-        await delay(25);
-        request.socket.resetAndDestroy();
-        return;
-      }
-      response.end(archive);
-    },
-  );
-  onTestFinished(close);
-  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
-  const address = server.address();
-  if (address === null || typeof address === "string")
-    throw new Error("Expected TCP fixture listener");
-
-  const workflow = z
-    .object({
-      jobs: z.object({
-        "real-web-network-captures": z.object({
-          steps: z.array(
-            z.object({
-              name: z.string().optional(),
-              run: z.string().optional(),
-            }),
-          ),
-        }),
-      }),
-    })
-    .parse(
-      parse(
-        await readFile(
-          new URL(
-            "../../../.github/workflows/real-web-network-captures.yml",
-            import.meta.url,
-          ),
-          "utf8",
-        ),
-      ),
-    );
-  const acquisition = workflow.jobs["real-web-network-captures"].steps.filter(
-    (step) =>
-      step.name ===
-      "Acquire fixed upstream native profile in runner temporary storage",
-  );
-  expect(acquisition).toHaveLength(1);
-  const source = z.string().parse(acquisition[0]?.run);
-  expect(source).toContain(upstream);
-  expect(source).toContain(digest);
-  // Execute the actual step with only its immutable input replaced by a tiny local archive.
-  const script = source
-    .replace(upstream, `http://127.0.0.1:${address.port}/profile.tar.gz`)
-    .replace(
-      digest,
-      outcome === "bad-digest"
-        ? "0".repeat(64)
-        : createHash("sha256").update(archive).digest("hex"),
-    );
-  const environment = join(directory, "github-env");
-  await writeFile(environment, "");
-  return {
-    get requests() {
-      return requests;
-    },
-    environment,
-    profile: join(directory, "rea-mitmproxy-12.2.3/mitmdump"),
-    run: () =>
-      executeWorkflowFixture(
-        process.execPath,
-        [
-          fileURLToPath(
-            new URL("../../../scripts/run-exclusive.mjs", import.meta.url),
-          ),
-          "--unlocked",
-          "bash",
-          "--noprofile",
-          "--norc",
-          "-e",
-          "-o",
-          "pipefail",
-          "-c",
-          script,
-        ],
-        {
-          cwd: directory,
-          env: {
-            ...process.env,
-            RUNNER_TEMP: directory,
-            GITHUB_ENV: environment,
-            CURL_HOME: directory,
-            NO_PROXY: "127.0.0.1",
-            no_proxy: "127.0.0.1",
-          },
-          timeout: 25_000,
-        },
-      ),
-  };
-}
-
-// This workflow runs on Ubuntu and requires these native tools; other hosts do not claim coverage.
-describe.runIf(process.platform === "linux")(
-  "native profile acquisition",
-  () => {
-    it("restarts a reset partial transfer, verifies the full archive and exports the extracted profile", async () => {
-      const download = await fixture("recover");
-      const result = await download.run();
-      expect(result.stderr).toContain("curl: (56)");
-      expect(download.requests).toBe(2);
-      expect(await readFile(download.profile, "utf8")).toBe(
-        "fixture native profile\n",
-      );
-      expect(await readFile(download.environment, "utf8")).toBe(
-        `REA_MITMDUMP_COMMAND=${download.profile}\n`,
-      );
-    });
-
-    it.each([
-      ["reset", 56, 4],
-      ["not-found", 22, 4],
-      ["bad-digest", 1, 1],
-    ] as const)(
-      "fails closed after %s without exporting a profile",
-      async (outcome, code, attempts) => {
-        const download = await fixture(outcome);
-        await expect(download.run()).rejects.toMatchObject({ code });
-        expect(download.requests).toBe(attempts);
-        await expect(readFile(download.profile)).rejects.toMatchObject({
-          code: "ENOENT",
-        });
-        expect(await readFile(download.environment, "utf8")).toBe("");
-      },
-    );
-  },
-);