chore: merge main into codex/audit-hopper-invocation-guidance

src/artifacts/pyinstaller/MarshalSubset.test.ts

@@ -20,11 +20,6 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
   ]);
 });
 
-it("reads marshal one-byte string tags as Latin-1", () => {
-  for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
-    expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
-});
-
 it("decodes CPython marshal Unicode with surrogatepass", () => {
   expect(
     readMarshalSubset(

src/artifacts/pyinstaller/MarshalSubset.ts

@@ -168,8 +168,14 @@ const readAsciiBytes = (
 ): Result<string, PyInstallerFailure> => {
   const bytes = readBytes(cursor, length);
   if (!bytes.ok) return bytes;
-  // These tags use CPython's one-byte Unicode constructor, not an ASCII check.
-  return ok(bytes.value.toString("latin1"));
+  if (bytes.value.some((byte) => byte > 0x7f))
+    return err(
+      pyInstallerFailure(
+        "format",
+        "PYZ marshal ASCII string contains a non-ASCII byte",
+      ),
+    );
+  return ok(bytes.value.toString("ascii"));
 };
 
 const readUtf8 = (

src/domain/analysisErrorPresentation.ts

@@ -73,7 +73,8 @@ export const analysisErrorRemediationAction = (
   error = resolvedAnalysisError(error);
   const continuing =
     callerLifetime === "continuing"
-      ? error.continuingCallerGuidance?.remediationAction
+      ? (error.continuingCallerGuidance?.remediationAction ??
+        hopperContinuingRecovery(error)?.remediationAction)
       : undefined;
   if (continuing !== undefined) return continuing;
   if (
@@ -162,11 +163,13 @@ export const analysisErrorRemediationAction = (
   if (error instanceof HopperTimeoutError)
     return error.operation === undefined
       ? HOPPER_STARTUP_TIMEOUT_REMEDIATION
-      : error.providerState === "busy"
-        ? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry."
-        : "Check binary_session for Hopper health, then retry the operation.";
+      : RERUN_HOPPER_INVOCATION;
   if (error instanceof HopperProcessError)
-    return hopperProcessRemediation(error);
+    return error.failureCode !== undefined && error.userMessage !== undefined
+      ? error.userMessage
+      : error.stage === "launch"
+        ? "Review the captured launcher diagnostics and Hopper setup, then retry opening the target."
+        : RERUN_HOPPER_INVOCATION;
   if (error instanceof HopperStartError)
     return error.ownerRunId === undefined
       ? "Check the Hopper launcher and target details, then retry opening the target."
@@ -427,10 +430,19 @@ const analysisErrorUserMessageParts = (
 ): { readonly outcome: string; readonly continuing?: string } => {
   error = resolvedAnalysisError(error);
   const outcome = outcomeUserMessage(error);
-  const continuing = error.continuingCallerGuidance?.userMessage;
+  const continuing =
+    error.continuingCallerGuidance?.userMessage ??
+    hopperContinuingRecovery(error)?.userMessage;
   return continuing === undefined ? { outcome } : { outcome, continuing };
 };
 
+/** Whether a child diagnostic needs a distinct continuing guidance snapshot. */
+export const hasAnalysisContinuingCallerGuidance = (
+  error: AnalysisError,
+): boolean =>
+  error.continuingCallerGuidance !== undefined ||
+  hopperContinuingRecovery(error) !== undefined;
+
 /**
  * Re-attribute one failure's message and another's remediation to a composed
  * error, keeping the form each caller lifetime receives.
@@ -550,8 +562,8 @@ const hopperErrorUserMessage = (error: AnalysisError): string | undefined => {
       return "REA timed out waiting for Hopper bridge readiness, which includes initial analysis. Hopper may still be analyzing the target or waiting for a loader, demo, or license dialog. Review the captured launcher outcome and resolve retained cleanup before retrying.";
     const request = error.operation;
     return error.providerState === "busy"
-      ? `Hopper timed out during ${request} while the provider remained busy. Check binary_session.analysis_activity, wait for the active request to finish, then retry.`
-      : `Hopper timed out during ${request} before it started. Check binary_session for provider health, then retry.`;
+      ? `Hopper timed out during ${request} while the provider remained busy.`
+      : `Hopper timed out during ${request} before it started.`;
   }
   if (error instanceof HopperProcessError) return hopperProcessMessage(error);
   if (error instanceof HopperStartError)
@@ -567,16 +579,38 @@ const hopperErrorUserMessage = (error: AnalysisError): string | undefined => {
 const RESTART_OWNED_PROVIDER =
   "Restart the owned provider by calling close_binary, then open the target again and retry.";
 const RESTART_UNREACHABLE_HOPPER = `${RESTART_OWNED_PROVIDER} close_binary preserves retained Evidence and Unknown history on this connection; export_evidence_bundle before ending the REA process if you need to resume later.`;
+const RERUN_HOPPER_INVOCATION =
+  "Review the reported Hopper state and resolve any incomplete cleanup before rerunning the command.";
 
-const hopperProcessRemediation = (error: HopperProcessError): string => {
-  if (error.failureCode !== undefined && error.userMessage !== undefined)
-    return error.userMessage;
-  if (error.stage === "launch")
-    return "Review the captured launcher diagnostics and Hopper setup, then retry opening the target.";
-  if (error.providerState === "unreachable") return RESTART_UNREACHABLE_HOPPER;
-  if (providerRetryAction(error.providerState) === "restart_provider")
-    return RESTART_OWNED_PROVIDER;
-  return "Read binary_session.provider_operation_health for this request before retrying or restarting the owned provider.";
+const hopperContinuingRecovery = (
+  error: AnalysisError,
+): AnalysisContinuingCallerGuidance | undefined => {
+  if (error instanceof HopperTimeoutError && error.operation !== undefined) {
+    const action =
+      error.providerState === "busy"
+        ? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry."
+        : "Check binary_session for Hopper health, then retry the operation.";
+    return { userMessage: action, remediationAction: action };
+  }
+  if (!(error instanceof HopperProcessError)) return undefined;
+  if (
+    error.providerState === "exited" &&
+    (error.stage === "launch" || error.failureCode !== undefined)
+  )
+    return undefined;
+  const action =
+    error.providerState === "unreachable"
+      ? RESTART_UNREACHABLE_HOPPER
+      : providerRetryAction(error.providerState) === "restart_provider"
+        ? RESTART_OWNED_PROVIDER
+        : "Read binary_session.provider_operation_health for this request before retrying or restarting the owned provider.";
+  return {
+    userMessage: action,
+    ...(error.stage === "launch" ||
+    (error.failureCode !== undefined && error.userMessage !== undefined)
+      ? {}
+      : { remediationAction: action }),
+  };
 };
 
 const hopperProcessMessage = (error: HopperProcessError): string => {
@@ -596,15 +630,11 @@ const hopperProcessMessage = (error: HopperProcessError): string => {
           ? "no exit code was observed"
           : `signal ${signal}`
         : `exit code ${String(error.exitCode)}`;
-    const recovery =
-      error.failureCode === undefined && stage !== "launch"
-        ? ` ${RESTART_OWNED_PROVIDER}`
-        : "";
-    return `Hopper exited during ${where} (${exit}).${startup}${recovery}${request}`;
+    return `Hopper exited during ${where} (${exit}).${startup}${request}`;
   }
   if (error.providerState === "unreachable")
-    return `Hopper became unreachable during ${where}; its process exit was not observed. ${RESTART_UNREACHABLE_HOPPER}${startup}${request}`;
-  return `Hopper failed during ${where} and REA could not determine whether the provider is busy, exited, or unreachable.${startup} Read binary_session.provider_operation_health for this request before retrying or restarting.${request}`;
+    return `Hopper became unreachable during ${where}; its process exit was not observed.${startup}${request}`;
+  return `Hopper failed during ${where} and REA could not determine whether the provider is busy, exited, or unreachable.${startup}${request}`;
 };
 
 const standardErrorMessage = (tag: AnalysisErrorTag): string | undefined => {

src/domain/analysisErrorProjection.test.ts

@@ -1569,11 +1569,91 @@ describe("analysis error projection: caller lifetime", () => {
       applies_at: "next_invocation",
       environment: "REA_HOPPER_STARTUP_TIMEOUT_MS",
     });
-    // A busy request is not a startup: it carries no lifetime-specific text.
-    const busy = new HopperTimeoutError(30_000, "trace_feature", 42, "busy");
-    expect(projectAnalysisError(busy, "invocation").remediation).toEqual(
-      projectAnalysisError(busy).remediation,
-    );
+  });
+});
+
+describe("analysis error projection: Hopper caller lifetime", () => {
+  it.each([
+    [
+      new HopperProcessError(7, { operation: "search_strings", requestId: 4 }),
+      "close_binary",
+      "exit code 7",
+    ],
+    [
+      new HopperProcessError(null, {
+        operation: "search_strings",
+        providerState: "unreachable",
+      }),
+      "export_evidence_bundle",
+      "process exit was not observed",
+    ],
+    [
+      new HopperProcessError(null, { operation: "search_strings" }),
+      "binary_session.provider_operation_health",
+      "could not determine",
+    ],
+    [
+      new HopperTimeoutError(30_000, "search_strings", 42, "busy"),
+      "binary_session.analysis_activity",
+      "remained busy",
+    ],
+    [
+      new HopperTimeoutError(30_000, "search_strings", 42, "not_started"),
+      "binary_session",
+      "before it started",
+    ],
+  ] as const)(
+    "keeps Hopper outcome facts and scopes request recovery to the caller",
+    (error, continuingTool, outcome) => {
+      const continuing = projectAnalysisError(error);
+      const invocation = projectAnalysisError(error, "invocation");
+      expect(continuing.message).toContain(continuingTool);
+      expect(continuing.remediation.action).toContain(continuingTool);
+      expect(invocation.message).toContain(outcome);
+      expect(invocation.message).not.toContain(continuingTool);
+      expect(invocation.remediation.action).toContain("rerunning the command");
+      expect(invocation.remediation.action).not.toContain(continuingTool);
+      expect(invocation.details).toEqual(continuing.details);
+      expect(invocation.code).toBe(continuing.code);
+      expect(invocation.category).toBe(continuing.category);
+      expect(invocation.retryable).toBe(continuing.retryable);
+      expect(analysisErrorProjectionSchema.parse(invocation)).toEqual(
+        invocation,
+      );
+    },
+  );
+
+  it("keeps Hopper recovery scoped through composed cleanup and its history", () => {
+    const primary = new HopperProcessError(null, {
+      operation: "search_strings",
+      providerState: "unreachable",
+    });
+    const cleanup = new ProviderCleanupError("hopper", ["hopper-bridge"], {});
+    const composed = analysisErrorWithCleanupFailure(
+      primary,
+      cleanup,
+    ).bindCleanupDependencies([cleanup]);
+    const invocationPrimary = projectAnalysisError(primary, "invocation");
+    const continuingPrimary = projectAnalysisError(primary);
+    const invocation = projectAnalysisError(composed, "invocation");
+    const continuing = projectAnalysisError(composed);
+    expect(invocation.message).toBe(invocationPrimary.message);
+    expect(continuing.message).toBe(continuingPrimary.message);
+    expect(invocation.details).toMatchObject({
+      resources: ["hopper-bridge"],
+      diagnostics: { primary_error: invocationPrimary },
+    });
+    expect(continuing.details).toMatchObject({
+      diagnostics: { primary_error: continuingPrimary },
+    });
+
+    cleanup.resolveCleanup();
+    expect(
+      projectAnalysisError(composed, "invocation").details?.cleanup_history,
+    ).toMatchObject({ primary_error: invocationPrimary });
+    expect(
+      projectAnalysisError(composed).details?.cleanup_history,
+    ).toMatchObject({ primary_error: continuingPrimary });
   });
 });
 

src/domain/analysisErrorProjection.ts

@@ -13,6 +13,7 @@ import {
   analysisErrorRemediationAction,
   analysisErrorUserMessage,
   assertKnownAnalysisErrorTag,
+  hasAnalysisContinuingCallerGuidance,
 } from "./analysisErrorPresentation.js";
 import {
   AnalysisAccessDeniedError,
@@ -318,7 +319,7 @@ const projectionDependsOnCallerLifetime = (
     )
       if (
         node.limit !== undefined ||
-        node.continuingCallerGuidance !== undefined ||
+        hasAnalysisContinuingCallerGuidance(node) ||
         node.invocationDiagnostics !== undefined ||
         node.workflowUnknownPublication !== undefined
       )

src/python/PythonPycParser.test.ts

@@ -407,30 +407,6 @@ it("charges long digits to the string budget and keeps large previews bounded",
   );
 });
 
-it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
-  for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
-    const encoded = Buffer.concat([
-      Buffer.from([tag]),
-      tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
-      Buffer.from([0xe9]),
-    ]);
-    const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
-    const set = Buffer.concat([
-      Buffer.from([0x3c]),
-      u32(2),
-      encoded,
-      sameUnicode,
-    ]);
-    const report = root(parse(withConsts([encoded, set])));
-    expect(report.consts[0]).toMatchObject({
-      kind: "str",
-      value: "é",
-      bytes: 1,
-    });
-    expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
-  }
-});
-
 it("decodes marshal strings with CPython surrogatepass rules", () => {
   const report = parse(
     withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
@@ -1558,36 +1534,6 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
   expect(pythonPycSchema.safeParse(report).success).toBe(true);
 });
 
-it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
-  const links = 3000;
-  const codes = Array.from({ length: links }, (_, index) =>
-    marshalCode("py311", {
-      flagged: true,
-      name: "f",
-      consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
-    }),
-  );
-  const bytes = Buffer.concat([
-    pythonPycHeader(3531),
-    marshalCode("py311", {
-      constsObject: Buffer.concat([
-        Buffer.from([0x28]),
-        u32(links + 1),
-        ...codes,
-        Buffer.from([0x3c]),
-        u32(1),
-        marshalRef(links - 1),
-      ]),
-    }),
-  ]);
-  const report = parseWith(bytes, { max_objects: 65_536 });
-  expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
-  expect(report.coverage).toMatchObject({
-    status: "complete",
-    code_objects: links + 1,
-  });
-});
-
 it("rejects code CPython treats as corrupt instead of complete coverage", () => {
   const file = (
     layout: "py37" | "py38" | "py311",

src/python/PythonPycParser.ts

@@ -332,7 +332,7 @@ class MarshalReader {
   readonly #codeKeys = new WeakMap<CodeNode, string>();
   readonly #codeIdentities = new WeakMap<CodeNode, number>();
   readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
-  /** Completed Python hashability checks per code object. */
+  /** Python hashability per code object. False while that check is in progress. */
   readonly #codeHashable = new WeakMap<CodeNode, boolean>();
   /** Tuples whose items are still being read; CPython cannot hash them yet. */
   readonly #readingTuples = new WeakSet<MarshalValue>();
@@ -436,19 +436,13 @@ class MarshalReader {
         };
       case TYPE_INTERNED:
       case TYPE_UNICODE:
-        return this.string(
-          this.nonNegativeInt32("Marshal string length"),
-          "utf8",
-        );
+        return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
       case TYPE_ASCII:
       case TYPE_ASCII_INTERNED:
-        return this.string(
-          this.nonNegativeInt32("Marshal string length"),
-          "latin1",
-        );
+        return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
       case TYPE_SHORT_ASCII:
       case TYPE_SHORT_ASCII_INTERNED:
-        return this.string(this.u8(), "latin1");
+        return this.utf8(this.u8(), true);
       case TYPE_TUPLE:
         return this.sequenceValue(
           "tuple",
@@ -1315,70 +1309,13 @@ class MarshalReader {
   #isCodeHashable(node: CodeNode): boolean {
     const known = this.#codeHashable.get(node);
     if (known !== undefined) return known;
-    const frames: { value: MarshalValue; finish: boolean }[] = [
-      { value: { kind: "code", value: node }, finish: false },
-    ];
-    const gray = new Set<MarshalValue | CodeNode>();
-    const complete = new Set<MarshalValue>();
-    const reject = (): false => {
-      this.#codeHashable.set(node, false);
-      return false;
-    };
-    while (frames.length > 0) {
-      this.#takeKeyStep();
-      const frame = frames.pop()!;
-      const value = frame.value;
-      const identity = value.kind === "code" ? value.value : value;
-      if (frame.finish) {
-        gray.delete(identity);
-        if (value.kind === "code") {
-          const material =
-            this.#codeEquality.get(value.value) ??
-            malformed("Code object is missing equality material.");
-          // Child code checks are now complete, so the existing tuple key
-          // walker preserves its equality and expansion budgets without recursion.
-          const hashable =
-            this.#equalityKey(material.constsTuple, "hash") !== null;
-          this.#codeHashable.set(value.value, hashable);
-          if (!hashable) return reject();
-        } else complete.add(value);
-        continue;
-      }
-      if (value.kind === "code") {
-        const cached = this.#codeHashable.get(value.value);
-        if (cached === false) return reject();
-        if (cached === true) continue;
-      } else {
-        const cached = this.#keyCache.get(value);
-        if (cached === null) return reject();
-        if (cached !== undefined || complete.has(value)) continue;
-      }
-      if (
-        gray.has(identity) ||
-        this.#readingTuples.has(value) ||
-        value.kind === "list" ||
-        value.kind === "set" ||
-        value.kind === "dict" ||
-        value.kind === "null"
-      )
-        return reject();
-      const children =
-        value.kind === "code"
-          ? [
-              (
-                this.#codeEquality.get(value.value) ??
-                malformed("Code object is missing equality material.")
-              ).constsTuple,
-            ]
-          : value.kind === "tuple" || value.kind === "frozenset"
-            ? value.items
-            : [];
-      gray.add(identity);
-      frames.push({ value, finish: true });
-      for (let index = children.length - 1; index >= 0; index -= 1)
-        frames.push({ value: children[index]!, finish: false });
-    }
-    return true;
+    this.#codeHashable.set(node, false);
+    const material =
+      this.#codeEquality.get(node) ??
+      malformed("Code object is missing equality material.");
+    const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
+    this.#codeHashable.set(node, hashable);
+    return hashable;
   }
 
   #codeIdentity(node: CodeNode): number {
@@ -1555,20 +1492,17 @@ class MarshalReader {
     );
   }
 
-  string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
+  utf8(length: number, ascii: boolean): MarshalValue {
     const bytes = this.blob(length);
-    if (encoding === "latin1") {
-      // CPython reads the ASCII-tagged forms as one-byte Unicode, even when
-      // high bytes appear. Identity must agree with the UTF-8 string form.
-      const value = bytes.toString("latin1");
+    if (ascii) {
+      if (bytes.some((byte) => byte > 0x7f))
+        malformed("Marshal ASCII string contains a non-ASCII byte.");
       return {
         kind: "str",
-        value,
+        value: bytes.toString("latin1"),
         previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
         bytes: length,
-        raw: bytes.some((byte) => byte > 0x7f)
-          ? Buffer.from(value, "utf8")
-          : bytes,
+        raw: bytes,
       };
     }
     const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);