feat(native): retain decompile export source facts (#8805)

* feat(native): retain decompile export source facts

* test(native): verify historical export source facts

* test(native): retain source after export conflict

docs/native-investigation.md

@@ -227,7 +227,17 @@ caveats. Failure details retain at most 32 rows within 2,048 JSON-encoded
 UTF-8 bytes; names may be omitted to keep complete addresses and reasons. The
 index retains every failure and its complete observed name.
 Each function's `decompile-index.json` record, whatever its status, keeps the
-complete limitations of the observations it used. The returned Evidence keeps
+complete limitations of the observations it used.
+When available, `provider_source` records the source of that attempt's
+pseudocode observation: a file path, SHA-256 and byte range, or a
+single-function decompile. The latter describes the method, not a unique run. A file
+descriptor records the source as observed; the file may since have changed or
+been removed, so compare its current digest before using it. The descriptor
+does not authenticate the journal. `status: "unknown"` distinguishes
+an unavailable source from an existing matching output whose original producer
+is unknown. Older journal records can omit this field; omission is unknown.
+Failed attempts retain their source in the append-only journal when the
+provider supplied one, even if a later attempt succeeds. The returned Evidence keeps
 at most eight first-observed distinct per-function provider limitations within an 8,000-byte JSON-encoded UTF-8 text budget and counts omitted observations. Repeated omitted text is counted per observation, not as a distinct caveat. This summary does not retain every omitted string in memory. A fresh output directory must not already exist. `--resume`
 continues an export for the same target SHA-256, provider, analysis profile,
 and layout; it requires the directory to exist, and an export without a known

src/application/AnalysisProvider.ts

@@ -18,6 +18,10 @@ import type {
   EvidenceSubjectTarget,
 } from "../domain/evidence.js";
 import { jsonValueSchema } from "../domain/jsonValue.js";
+import {
+  analysisSourceDescriptorSchema,
+  type AnalysisSourceDescriptor,
+} from "../domain/analysisSource.js";
 import type { ProgressReporter } from "./ProgressReporter.js";
 import type { ProviderRejectionCode } from "../domain/providerSelection.js";
 import type { ProcessLineageObservation } from "../process/ProcessOwnership.js";
@@ -60,6 +64,12 @@ export type AnalysisOperation =
 /** Atomic successful provider observation with exact producing provenance. */
 export interface AnalysisExecution extends AnalysisObservation {
   readonly provider: ProviderIdentity;
+  /**
+   * Compact provider-owned facts for this execution. This is not an Evidence
+   * record and is not restored by snapshot replay; consumers must treat its
+   * absence as unknown.
+   */
+  readonly sourceDescriptor?: AnalysisSourceDescriptor;
   /** Session-issued identity of the committed record consumed by this execution. */
   readonly evidenceId?: string;
   /** Result-specific veto of snapshot replay; Evidence remains retainable. */
@@ -75,6 +85,7 @@ export interface AnalysisExecution extends AnalysisObservation {
 
 interface CreateAnalysisExecutionOptions {
   readonly rawResult?: unknown;
+  readonly sourceDescriptor?: AnalysisSourceDescriptor;
   readonly analysisProfile?: AnalysisProfileCommitment;
   readonly cachePolicy?: "live";
   readonly limitations?: readonly string[];
@@ -114,6 +125,13 @@ export function createAnalysisExecution(
         ? null
         : jsonValueSchema.parse(options.rawResult),
     provider,
+    ...(options.sourceDescriptor === undefined
+      ? {}
+      : {
+          sourceDescriptor: analysisSourceDescriptorSchema.parse(
+            options.sourceDescriptor,
+          ),
+        }),
     ...(options.cachePolicy === undefined
       ? {}
       : { cachePolicy: options.cachePolicy }),

src/application/decompileDirectory/decompileAll.ts

@@ -535,6 +535,12 @@ const publishOne = async (input: {
   const caveats = [
     ...new Set([...infoCaveats, ...decompiled.value.limitations]),
   ];
+  const observationSource: NonNullable<
+    DecompileDirectoryFunctionRecord["provider_source"]
+  > =
+    decompiled.value.sourceDescriptor === undefined
+      ? { status: "unknown", reason: "source_not_available" }
+      : { status: "reported", source: decompiled.value.sourceDescriptor };
   const pseudocode = decompiled.value.result;
   if (typeof pseudocode !== "string" || pseudocode.length === 0) {
     const reason = pseudocodeFailure(pseudocode);
@@ -543,6 +549,7 @@ const publishOne = async (input: {
       result: await input.session.append(
         failureRecord(input.procedure, body.fact, reason, {
           limitations: caveats,
+          providerSource: observationSource,
         }),
       ),
       bodySupport: body.support,
@@ -580,6 +587,7 @@ const publishOne = async (input: {
         failureRecord(input.procedure, body.fact, reason, {
           relativePath,
           limitations: caveats,
+          providerSource: observationSource,
         }),
       ),
       bodySupport: body.support,
@@ -591,6 +599,11 @@ const publishOne = async (input: {
     status: "written",
     relative_path: relativePath,
     sha256: committed.sha256,
+    // Matching existing bytes prove equality, not who originally wrote them.
+    provider_source:
+      committed.status === "matched"
+        ? { status: "unknown", reason: "existing_file_origin_unknown" }
+        : observationSource,
     body: bodyRecord(body.fact),
     // The header may truncate; the journal keeps the complete list.
     ...(caveats.length === 0 ? {} : { limitations: caveats }),
@@ -889,6 +902,7 @@ const failureRecord = (
     readonly relativePath?: string;
     /** Caveats of the observations this attempt used. */
     readonly limitations?: readonly string[];
+    readonly providerSource?: DecompileDirectoryFunctionRecord["provider_source"];
   } = {},
 ): DecompileDirectoryFunctionRecord => ({
   address: procedure.address,
@@ -896,6 +910,9 @@ const failureRecord = (
   status: "failed",
   body: bodyRecord(body),
   failure: reason,
+  ...(options.providerSource === undefined
+    ? {}
+    : { provider_source: options.providerSource }),
   ...(options.relativePath === undefined
     ? {}
     : { relative_path: options.relativePath }),

src/contracts/decompileDirectory.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 
 import { digestSchema } from "../domain/digests.js";
 import { absoluteLocalPathStringSchema } from "../domain/localPath.js";
+import { analysisSourceDescriptorSchema } from "../domain/analysisSource.js";
 
 /** On-disk journal. Append-only so a crash keeps committed function files. */
 export const DECOMPILE_DIRECTORY_JOURNAL_NAME = "decompile-journal.jsonl";
@@ -245,6 +246,22 @@ export const decompileDirectoryFunctionSchema = z
       )
       .optional(),
     sha256: digestSchema.optional(),
+    /** Source of this attempt's pseudocode observation; a matched file's original producer remains unknown. */
+    provider_source: z
+      .discriminatedUnion("status", [
+        z.strictObject({
+          status: z.literal("reported"),
+          source: analysisSourceDescriptorSchema,
+        }),
+        z.strictObject({
+          status: z.literal("unknown"),
+          reason: z.enum([
+            "source_not_available",
+            "existing_file_origin_unknown",
+          ]),
+        }),
+      ])
+      .optional(),
     body: bodySchema,
     failure: z
       .strictObject({

src/domain/analysisSource.ts

@@ -0,0 +1,19 @@
+import { z } from "zod";
+import { digestSchema } from "./digests.js";
+import { absoluteLocalPathStringSchema } from "./localPath.js";
+
+/** Compact source facts supplied by the provider for one observation. */
+export const analysisSourceDescriptorSchema = z.discriminatedUnion("kind", [
+  z.strictObject({
+    kind: z.literal("file-range"),
+    path: absoluteLocalPathStringSchema,
+    sha256: digestSchema,
+    byte_offset: z.int().nonnegative(),
+    byte_length: z.int().nonnegative(),
+  }),
+  z.strictObject({ kind: z.literal("single-function-decompile") }),
+]);
+
+export type AnalysisSourceDescriptor = z.infer<
+  typeof analysisSourceDescriptorSchema
+>;

src/kuna/KunaSessionClient.ts

@@ -1,5 +1,5 @@
 import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js";
-import { basename } from "node:path";
+import { basename, join } from "node:path";
 import {
   regularFileCancellation,
   RegularFileCleanupFailure,
@@ -624,6 +624,30 @@ export class KunaSessionClient implements AnalysisClient {
     // component once, including its producing file digest and byte range.
     const recordedFiles =
       files.length === 1 ? files[0] : files.length === 0 ? {} : { files };
+    const rawSource =
+      source === null
+        ? undefined
+        : source.kind === "single-function"
+          ? { kind: "single-function" }
+          : {
+              kind: "project-export",
+              key: source.project.key,
+              directory: source.project.directory,
+              ...recordedFiles,
+            };
+    const file = files.length === 1 ? files[0] : undefined;
+    const sourceDescriptor =
+      source?.kind === "single-function"
+        ? { kind: "single-function-decompile" as const }
+        : source?.kind === "project-export" && file?.file_sha256 != null
+          ? {
+              kind: "file-range" as const,
+              path: join(source.project.directory, file.file),
+              sha256: file.file_sha256,
+              byte_offset: file.byte_offset,
+              byte_length: file.byte_length,
+            }
+          : undefined;
     return createAnalysisExecution(
       answer.result,
       {
@@ -632,6 +656,7 @@ export class KunaSessionClient implements AnalysisClient {
         version: installation.version,
       },
       {
+        ...(sourceDescriptor === undefined ? {} : { sourceDescriptor }),
         // Kuna's identity is in the analysis profile; the raw result records
         // only where file-backed text came from.
         ...(source === null &&
@@ -643,15 +668,7 @@ export class KunaSessionClient implements AnalysisClient {
                 ...(source === null
                   ? {}
                   : {
-                      source:
-                        source.kind === "single-function"
-                          ? { kind: "single-function" }
-                          : {
-                              kind: "project-export",
-                              key: source.project.key,
-                              directory: source.project.directory,
-                              ...recordedFiles,
-                            },
+                      source: rawSource,
                       project_export: this.#exportState(),
                     }),
                 ...(answer.referenceSource === undefined

tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts

@@ -1835,3 +1835,65 @@ it("retains a native dispatch workflow snapshot without a deep provider profile"
     saved.value,
   );
 });
+
+it("runs decompile export against the provider after loading a pseudocode snapshot", async () => {
+  const fixture = await createSnapshotReplayFixture("rea-directory-snapshot-");
+  const original = fixture.createProvider({
+    supportedOperations: ["list_procedures", "procedure_pseudo_code"],
+  });
+  const provider: AnalysisProvider = {
+    ...original,
+    createClient: (...args) => {
+      const client = original.createClient(...args);
+      return {
+        ...client,
+        execute: async (operation, parameters, options) => {
+          const result = await client.execute(operation, parameters, options);
+          return result.ok && operation === "procedure_pseudo_code"
+            ? ok({
+                ...result.value,
+                sourceDescriptor: {
+                  kind: "single-function-decompile" as const,
+                },
+              })
+            : result;
+        },
+      };
+    },
+  };
+  const dependencies = fixture.dependencies(provider);
+  const first = await runDirectAnalysis(
+    dependencies,
+    fixture.path,
+    "procedure_pseudo_code",
+    { procedure: "0x1000", document: fixture.path },
+    { snapshotPath: fixture.snapshotPath },
+  );
+  expect(first.ok).toBe(true);
+  const snapshot = await readAnalysisSnapshot(fixture.snapshotPath);
+  if (!snapshot.ok) throw snapshot.error;
+  expect(
+    snapshot.value.entries.some(
+      (entry) => entry.operation === "procedure_pseudo_code",
+    ),
+  ).toBe(true);
+  const directory = join(dirname(fixture.snapshotPath), "export");
+  const exported = await runDirectAnalysis(
+    dependencies,
+    fixture.path,
+    "decompile_all",
+    { output_directory: directory, layout: "address", resume: false },
+    { snapshotPath: fixture.snapshotPath },
+  );
+  expect(exported.ok, JSON.stringify(exported)).toBe(true);
+  expect(
+    fixture.calls.filter((name) => name === "procedure_pseudo_code"),
+  ).toHaveLength(2);
+  const index = JSON.parse(
+    await readFile(join(directory, "decompile-index.json"), "utf8"),
+  ) as { functions: Array<{ provider_source?: unknown }> };
+  expect(index.functions[0]?.provider_source).toEqual({
+    status: "reported",
+    source: { kind: "single-function-decompile" },
+  });
+});

tests/boundary/providers/kuna/kunaDirectoryProvenanceMcp.test.ts

@@ -0,0 +1,172 @@
+import { createHash } from "node:crypto";
+import { appendFile, readFile, unlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { expect, it as test } from "vitest";
+import { AnalysisProviderRegistry } from "../../../../src/application/binary/AnalysisProviderRegistry.js";
+import { composeBinarySession } from "../../../../src/application/binary/BinarySessionComposition.js";
+import { decompileDirectoryIndexSchema } from "../../../../src/contracts/decompileDirectory.js";
+import { parseEvidence } from "../../../../src/domain/evidence.js";
+import { KunaProvider } from "../../../../src/kuna/KunaProvider.js";
+import { elfNoteFixture } from "../../../../src/native/elf/ElfNotes.fixture.js";
+import { createApplicationMcpHarness } from "../../../fixtures/applicationMcpHarness.js";
+import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js";
+import { setFakeKunaBehavior, writeFakeKuna } from "./fakeKuna.js";
+import { FUNCTIONS } from "./kunaSessionFixture.js";
+
+const it = test.skipIf(process.platform === "win32");
+
+it("retains each written function's producer across an export resume", async () => {
+  const root = await createTestTempDirectory("rea-kuna-directory-provenance-");
+  const kuna = await writeFakeKuna(root, { functions: FUNCTIONS });
+  const target = join(root, "program");
+  const output = join(root, "output");
+  await writeFile(target, elfNoteFixture({ bits: 64, type: 2 }));
+  const run = async (cache: string, resume: boolean) => {
+    const provider = new KunaProvider(
+      { REA_KUNA_COMMAND: kuna, REA_KUNA_CACHE_DIR: join(root, cache) },
+      { platform: process.platform },
+    );
+    const session = composeBinarySession(
+      new AnalysisProviderRegistry([provider], "kuna"),
+    );
+    const harness = await createApplicationMcpHarness({ session });
+    try {
+      const call = async (name: string, args: Record<string, unknown>) => {
+        const response = await harness.client.callTool({
+          name,
+          arguments: args,
+        });
+        expect(response.isError, JSON.stringify(response)).not.toBe(true);
+        return parseEvidence(response.structuredContent);
+      };
+      const opened = await harness.client.callTool({
+        name: "open_binary",
+        arguments: { path: target, provider_id: "kuna" },
+      });
+      expect(opened.isError).not.toBe(true);
+      await call("procedure_assembly", { procedure: "main" });
+      const result = await call("decompile_all", {
+        output_directory: output,
+        resume,
+      });
+      const resultValue = result.normalized_result as { index_path: string };
+      return decompileDirectoryIndexSchema.parse(
+        JSON.parse(await readFile(resultValue.index_path, "utf8")),
+      );
+    } finally {
+      await harness.close();
+      const closed = await provider.closeOwnedResources();
+      if (!closed.ok) throw closed.error;
+    }
+  };
+  const partial = await run("partial", false);
+  const first = partial.functions.find((item) => item.address === "0x1000");
+  const failed = partial.functions.find((item) => item.address === "0x1080");
+  expect(first?.status).toBe("written");
+  expect(first?.provider_source).toMatchObject({
+    status: "reported",
+    source: { kind: "file-range" },
+  });
+  expect(failed?.status).toBe("failed");
+  expect(failed?.provider_source).toMatchObject({
+    status: "reported",
+    source: { kind: "file-range" },
+  });
+  if (first?.relative_path === undefined)
+    throw new Error("missing initial file");
+  const digest = (value: Uint8Array) =>
+    createHash("sha256").update(value).digest("hex");
+  const before = digest(await readFile(join(output, first.relative_path)));
+  // Simulate an interrupted append after the committed failed observation.
+  await appendFile(join(output, "decompile-journal.jsonl"), '{"kind":"func');
+  await setFakeKunaBehavior(kuna, {
+    functions: FUNCTIONS.map(({ name, addr }) => ({ name, addr })),
+  });
+  const clean = await run("clean", true);
+  const kept = clean.functions.find((item) => item.address === "0x1000");
+  const recovered = clean.functions.find((item) => item.address === "0x1080");
+  expect(kept).toEqual(first);
+  expect(digest(await readFile(join(output, first.relative_path)))).toBe(
+    before,
+  );
+  expect(recovered?.status).toBe("written");
+  expect(recovered?.provider_source).toMatchObject({
+    status: "reported",
+    source: { kind: "file-range" },
+  });
+  if (recovered === undefined) throw new Error("missing recovered function");
+  const oldSource = first.provider_source;
+  const newSource = recovered.provider_source;
+  if (
+    oldSource?.status !== "reported" ||
+    oldSource.source.kind !== "file-range" ||
+    newSource?.status !== "reported" ||
+    newSource.source.kind !== "file-range"
+  )
+    throw new Error("missing source file ranges");
+  expect(newSource.source.sha256).not.toBe(oldSource.source.sha256);
+  const journal = (
+    await readFile(join(output, "decompile-journal.jsonl"), "utf8")
+  )
+    .trim()
+    .split("\n")
+    .map(
+      (line) =>
+        JSON.parse(line) as {
+          address?: string;
+          status?: string;
+          provider_source?: unknown;
+        },
+    );
+  expect(journal).toContainEqual(
+    expect.objectContaining({
+      address: "0x1080",
+      status: "failed",
+      provider_source: failed?.provider_source,
+    }),
+  );
+  // The descriptor is historical: the source cache can disappear while the
+  // verified output and its recorded source facts remain unchanged.
+  await unlink(oldSource.source.path);
+  const afterSourceRemoval = await run("clean", true);
+  expect(
+    afterSourceRemoval.functions.find((item) => item.address === "0x1000"),
+  ).toEqual(first);
+  expect(digest(await readFile(join(output, first.relative_path)))).toBe(
+    before,
+  );
+  const outputPath = join(output, first.relative_path);
+  const originalBytes = await readFile(outputPath);
+  await writeFile(outputPath, "owned modified output");
+  const conflict = await run("clean", true);
+  const conflictMain = conflict.functions.find(
+    (item) => item.address === "0x1000",
+  );
+  expect(conflictMain).toMatchObject({
+    status: "failed",
+    failure: { code: "output_exists" },
+  });
+  expect(conflictMain?.provider_source).toBeUndefined();
+  expect(await readFile(outputPath, "utf8")).toBe("owned modified output");
+  await writeFile(outputPath, originalBytes);
+  const restored = await run("clean", true);
+  expect(restored.functions.find((item) => item.address === "0x1000")).toEqual(
+    first,
+  );
+  // A file published before a journal append can be adopted on resume, but
+  // matching bytes do not identify the run that originally wrote the file.
+  const journalPath = join(output, "decompile-journal.jsonl");
+  const header = (await readFile(journalPath, "utf8")).split("\n")[0];
+  await writeFile(journalPath, `${header}\n`);
+  const adopted = await run("clean", true);
+  const adoptedMain = adopted.functions.find(
+    (item) => item.address === "0x1000",
+  );
+  expect(adoptedMain?.provider_source).toEqual({
+    status: "unknown",
+    reason: "existing_file_origin_unknown",
+  });
+  expect(digest(await readFile(join(output, first.relative_path)))).toBe(
+    before,
+  );
+});