feat(memory): phase 6 — post-turn extraction primitives (opt-in)

- MemoryExtractor: provider-agnostic via ModelComplete adapter, fence-strip, JSON parse, normalization, max 3 proposals
- PendingStore: persisted JSON in .soulforge/memory-pending.json, atomic write
- MemoryManager: addPending/listPending/acceptPending/rejectPending/clearPending
- memory tool: list_pending / accept_pending / reject_pending actions
- AppConfig: memory.postTurnExtraction.{enabled,model,maxPerTurn} (default disabled)
- tests: parser robustness, persistence, accept-writes-real-memory, reject-discards
- agent loop hook for auto-trigger left as follow-up (foundation ready)

Co-Authored-By: SoulForge <[email protected]>

src/core/memory/extractor.ts

@@ -0,0 +1,151 @@
+/**
+ * Phase 6 — post-turn extraction (opt-in).
+ *
+ * After an agent turn finishes, optionally scan the user's prompt + assistant
+ * reply for durable, non-code knowledge worth remembering, and propose
+ * candidate memories to the user. The model never writes directly — it
+ * returns a JSON array; the user confirms via UI/CLI.
+ *
+ * Contract:
+ * - Disabled by default. AppConfig.memory.postTurnExtraction.enabled gates it.
+ * - Provider-agnostic: caller supplies a `complete(prompt) => Promise<string>`
+ *   adapter so we don't import the LLM stack here.
+ * - Failure modes (network, timeout, JSON parse, empty) all collapse to []
+ *   — extraction is best-effort, never blocking.
+ */
+
+import type { MemoryCategory } from "./types.js";
+
+export interface ExtractedProposal {
+  summary: string;
+  details: string;
+  category: MemoryCategory | null;
+  topics: string[];
+  file_paths: string[];
+}
+
+export interface PendingProposal extends ExtractedProposal {
+  id: string;
+  proposed_at: string;
+  source_session_id: string | null;
+  source_turn_index: number | null;
+}
+
+export type ModelComplete = (prompt: string) => Promise<string>;
+
+const PROMPT = `You will be given a user message and an assistant reply from a coding session.
+Extract at most 3 DURABLE, NON-CODE memories worth remembering across sessions.
+
+What counts:
+- user preferences ("user wants terse output")
+- decisions with rationale ("we use bun, not node, because…")
+- gotchas ("jwt.ts secrets must rotate every 90d")
+- project context ("legacy/ is replaced by core/ next sprint")
+
+What does NOT count:
+- anything visible from the codebase (file/symbol/signature) — Soul Map handles those
+- one-shot facts that won't matter next session
+- the conversation itself
+
+Return ONLY a JSON array. Each element:
+{
+  "summary": "<≤200ch headline>",
+  "details": "<≤2000ch context, can be empty>",
+  "category": "pref" | "decision" | "gotcha" | "context" | null,
+  "topics": ["<≤8 tags>"],
+  "file_paths": ["<relative paths if specifically about these files>"]
+}
+
+If nothing qualifies, return []. No prose. No markdown fence. Just the array.
+
+---
+USER:
+%USER%
+
+---
+ASSISTANT:
+%ASSISTANT%
+---`;
+
+const SUMMARY_MAX = 200;
+const DETAILS_MAX = 2000;
+const TOPICS_MAX = 8;
+const FILES_MAX = 16;
+
+export class MemoryExtractor {
+  constructor(private readonly complete: ModelComplete) {}
+
+  async proposeFromTurn(
+    userMessage: string,
+    assistantMessage: string,
+  ): Promise<ExtractedProposal[]> {
+    const prompt = PROMPT.replace("%USER%", userMessage.trim()).replace(
+      "%ASSISTANT%",
+      assistantMessage.trim(),
+    );
+    let raw: string;
+    try {
+      raw = await this.complete(prompt);
+    } catch {
+      return [];
+    }
+    return parseProposals(raw);
+  }
+}
+
+export function parseProposals(raw: string): ExtractedProposal[] {
+  const trimmed = stripFence(raw).trim();
+  if (!trimmed) return [];
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(trimmed);
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const out: ExtractedProposal[] = [];
+  for (const item of parsed) {
+    const norm = normalizeProposal(item);
+    if (norm) out.push(norm);
+    if (out.length >= 3) break;
+  }
+  return out;
+}
+
+function stripFence(s: string): string {
+  // Models occasionally wrap in ```json ... ``` even when told not to.
+  const fenced = s.match(/```(?:json)?\s*([\s\S]*?)```/);
+  return fenced ? (fenced[1] ?? s) : s;
+}
+
+function normalizeProposal(item: unknown): ExtractedProposal | null {
+  if (!item || typeof item !== "object") return null;
+  const o = item as Record<string, unknown>;
+  const summary = typeof o.summary === "string" ? o.summary.trim().slice(0, SUMMARY_MAX) : "";
+  if (!summary) return null;
+  const details = typeof o.details === "string" ? o.details.trim().slice(0, DETAILS_MAX) : "";
+  const category = isCategory(o.category) ? o.category : null;
+  const topics: string[] = [];
+  if (Array.isArray(o.topics)) {
+    for (const t of o.topics) {
+      if (typeof t !== "string") continue;
+      const trimmed = t.trim().slice(0, 32);
+      if (trimmed) topics.push(trimmed);
+      if (topics.length >= TOPICS_MAX) break;
+    }
+  }
+  const filePaths: string[] = [];
+  if (Array.isArray(o.file_paths)) {
+    for (const p of o.file_paths) {
+      if (typeof p !== "string") continue;
+      const trimmed = p.trim();
+      if (trimmed) filePaths.push(trimmed);
+      if (filePaths.length >= FILES_MAX) break;
+    }
+  }
+  return { summary, details, category, topics, file_paths: filePaths };
+}
+
+function isCategory(v: unknown): v is MemoryCategory {
+  return v === "pref" || v === "decision" || v === "gotcha" || v === "context";
+}

src/core/memory/manager.ts

@@ -7,6 +7,8 @@ import {
   type MemoryWriteInput,
   type MemoryWriteResult,
 } from "./db.js";
+import type { ExtractedProposal, PendingProposal } from "./extractor.js";
+import { PendingStore } from "./pending.js";
 import type {
   MemoryCategory,
   MemoryFileRef,
@@ -413,4 +415,63 @@ export class MemoryManager {
     if (n > 0) this._generation++;
     return n;
   }
+
+  private _pending: PendingStore | null = null;
+
+  private pending(): PendingStore {
+    if (!this._pending) this._pending = new PendingStore(this.cwd);
+    return this._pending;
+  }
+
+  /** Phase 6 — list pending extraction proposals (project-scoped). */
+  listPending(): PendingProposal[] {
+    return this.pending().list();
+  }
+
+  addPending(
+    proposals: ExtractedProposal[],
+    sessionId: string | null,
+    turnIndex: number | null,
+  ): PendingProposal[] {
+    const now = new Date().toISOString();
+    const added: PendingProposal[] = [];
+    for (const p of proposals) {
+      const item: PendingProposal = {
+        ...p,
+        id: crypto.randomUUID(),
+        proposed_at: now,
+        source_session_id: sessionId,
+        source_turn_index: turnIndex,
+      };
+      this.pending().add(item);
+      added.push(item);
+    }
+    return added;
+  }
+
+  acceptPending(id: string, scope: MemoryScope): MemoryRecord | null {
+    const p = this.pending().get(id);
+    if (!p) return null;
+    const result = this.write(scope, {
+      summary: p.summary,
+      details: p.details,
+      topics: p.topics,
+      category: p.category,
+      source: "agent",
+      session_id: p.source_session_id ?? null,
+    });
+    for (const path of p.file_paths) {
+      this.addFileRef(scope, result.record.id, path, null);
+    }
+    this.pending().remove(id);
+    return result.record;
+  }
+
+  rejectPending(id: string): boolean {
+    return this.pending().remove(id);
+  }
+
+  clearPending(): number {
+    return this.pending().clear();
+  }
 }

src/core/memory/pending.ts

@@ -0,0 +1,97 @@
+/**
+ * Pending-proposals store. Persists Phase 6 extraction proposals on disk
+ * so they survive process restarts. User accepts/rejects via UI; only on
+ * accept does the proposal become a real memory.
+ *
+ * Single JSON file per scope (project) — small N (≤50 entries), no need
+ * for SQLite. Atomic write via tmp+rename.
+ */
+
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import type { PendingProposal } from "./extractor.js";
+
+const FILE = "memory-pending.json";
+const MAX_PENDING = 50;
+
+export class PendingStore {
+  private items: PendingProposal[] = [];
+  private path: string;
+  private loaded = false;
+
+  constructor(cwd: string) {
+    this.path = join(cwd, ".soulforge", FILE);
+  }
+
+  private ensureLoaded(): void {
+    if (this.loaded) return;
+    this.loaded = true;
+    if (!existsSync(this.path)) return;
+    try {
+      const raw = readFileSync(this.path, "utf-8");
+      const parsed = JSON.parse(raw) as unknown;
+      if (Array.isArray(parsed)) {
+        this.items = parsed.filter(isPendingProposal);
+      }
+    } catch {
+      this.items = [];
+    }
+  }
+
+  list(): PendingProposal[] {
+    this.ensureLoaded();
+    return [...this.items];
+  }
+
+  add(p: PendingProposal): void {
+    this.ensureLoaded();
+    this.items.unshift(p);
+    if (this.items.length > MAX_PENDING) this.items.length = MAX_PENDING;
+    this.flush();
+  }
+
+  remove(id: string): boolean {
+    this.ensureLoaded();
+    const before = this.items.length;
+    this.items = this.items.filter((p) => p.id !== id);
+    if (this.items.length === before) return false;
+    this.flush();
+    return true;
+  }
+
+  get(id: string): PendingProposal | null {
+    this.ensureLoaded();
+    return this.items.find((p) => p.id === id) ?? null;
+  }
+
+  clear(): number {
+    this.ensureLoaded();
+    const n = this.items.length;
+    this.items = [];
+    this.flush();
+    return n;
+  }
+
+  private flush(): void {
+    try {
+      const dir = dirname(this.path);
+      if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+      const tmp = `${this.path}.tmp`;
+      writeFileSync(tmp, JSON.stringify(this.items, null, 2), "utf-8");
+      renameSync(tmp, this.path);
+    } catch {}
+  }
+}
+
+function isPendingProposal(v: unknown): v is PendingProposal {
+  if (!v || typeof v !== "object") return false;
+  const o = v as Record<string, unknown>;
+  return (
+    typeof o.id === "string" &&
+    typeof o.summary === "string" &&
+    typeof o.details === "string" &&
+    Array.isArray(o.topics) &&
+    Array.isArray(o.file_paths) &&
+    typeof o.proposed_at === "string"
+  );
+}

src/core/tools/memory.ts

@@ -48,7 +48,19 @@ export function createMemoryTool(deps: MemoryManager | CreateMemoryToolDeps) {
     description:
       "Persistent memory across sessions. Memory surfaces relevant entries automatically when you edit or mention related files — explicit search is only for targeted lookup. Write durable, non-code knowledge: user preferences, decisions with rationale, gotchas, project history. Don't store anything the codebase already shows (Soul Map covers that).",
     inputSchema: z.object({
-      action: z.enum(["write", "get", "list", "search", "delete", "restore", "pin", "unpin"]),
+      action: z.enum([
+        "write",
+        "get",
+        "list",
+        "search",
+        "delete",
+        "restore",
+        "pin",
+        "unpin",
+        "list_pending",
+        "accept_pending",
+        "reject_pending",
+      ]),
       scope: z
         .enum(["global", "project", "both", "all"])
         .nullable()
@@ -110,6 +122,12 @@ export function createMemoryTool(deps: MemoryManager | CreateMemoryToolDeps) {
             return handlePin(true);
           case "unpin":
             return handlePin(false);
+          case "list_pending":
+            return handleListPending();
+          case "accept_pending":
+            return handleAcceptPending();
+          case "reject_pending":
+            return handleRejectPending();
           default:
             return {
               success: false,
@@ -304,6 +322,48 @@ export function createMemoryTool(deps: MemoryManager | CreateMemoryToolDeps) {
         return { success: true, output: `Restored ${args.id.slice(0, 8)}` };
       }
 
+      function handleListPending() {
+        const items = manager.listPending();
+        if (items.length === 0) return { success: true, output: "No pending memory proposals." };
+        const lines = items.map(
+          (p) =>
+            `[${p.id.slice(0, 8)}] ${p.category ?? "—"} | ${p.summary}${
+              p.topics.length > 0 ? ` (topics: ${p.topics.join(", ")})` : ""
+            }`,
+        );
+        return { success: true, output: lines.join("\n"), data: { pending: items } };
+      }
+
+      function handleAcceptPending() {
+        if (!args.id)
+          return { success: false, output: "id required for accept_pending", error: "missing_id" };
+        const writeScope = resolveWriteScope(args.scope);
+        if (writeScope === "disabled" || writeScope === "invalid") {
+          return {
+            success: false,
+            output: "scope must be 'project' or 'global'",
+            error: "bad_scope",
+          };
+        }
+        const record = manager.acceptPending(args.id, writeScope);
+        if (!record)
+          return { success: false, output: `Pending not found: ${args.id}`, error: "not_found" };
+        return {
+          success: true,
+          output: `Accepted: "${record.summary}" (${record.id.slice(0, 8)}, ${writeScope})`,
+          data: { id: record.id, scope: writeScope },
+        };
+      }
+
+      function handleRejectPending() {
+        if (!args.id)
+          return { success: false, output: "id required for reject_pending", error: "missing_id" };
+        const ok = manager.rejectPending(args.id);
+        if (!ok)
+          return { success: false, output: `Pending not found: ${args.id}`, error: "not_found" };
+        return { success: true, output: `Rejected pending ${args.id.slice(0, 8)}` };
+      }
+
       function handlePin(pin: boolean) {
         if (!args.id) {
           return { success: false, output: "id required", error: "missing_id" };

src/types/index.ts

@@ -352,6 +352,19 @@ export interface AppConfig {
   toolTimeout?: number;
   /** Retry behavior for transient provider errors (429, 529, 503, timeouts, overloaded). */
   retry?: RetryConfig;
+  /** Memory subsystem config. */
+  memory?: MemoryConfig;
+}
+
+export interface MemoryConfig {
+  /** Phase 6 — post-turn memory extraction. Default: disabled. */
+  postTurnExtraction?: {
+    enabled?: boolean;
+    /** Optional model override (defaults to taskRouter.semantic, then defaultModel). */
+    model?: string;
+    /** Cap on proposals per turn. Default: 3 */
+    maxPerTurn?: number;
+  };
 }
 
 export interface RetryConfig {

tests/memory-phase4.test.ts

@@ -224,7 +224,7 @@ describe("MemoryDB — schema v2 migration", () => {
       const missing = db.listMissingEmbeddings(undefined, 10);
       expect(missing.length).toBe(1);
       const linked = db.embedAndLink(a.record.id);
-      expect(linked).toBe(0); // no other memories
+      expect(linked).toEqual([]); // no other memories to link
       expect(db.getEmbedding(a.record.id)).not.toBeNull();
     } finally {
       db.close();

tests/memory-phase6.test.ts

@@ -0,0 +1,194 @@
+import { mkdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { MemoryExtractor, parseProposals } from "../src/core/memory/extractor.js";
+import { MemoryManager } from "../src/core/memory/manager.js";
+import { PendingStore } from "../src/core/memory/pending.js";
+
+function makeTmpDir(label: string): string {
+  const dir = join(
+    tmpdir(),
+    `mem-phase6-${label}-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(dir, { recursive: true });
+  mkdirSync(join(dir, ".soulforge"), { recursive: true });
+  return dir;
+}
+
+describe("MemoryExtractor — parsing", () => {
+  it("parses plain JSON array", () => {
+    const raw = `[{"summary":"use bun","details":"","category":"decision","topics":["tooling"],"file_paths":[]}]`;
+    const out = parseProposals(raw);
+    expect(out).toHaveLength(1);
+    expect(out[0]!.summary).toBe("use bun");
+    expect(out[0]!.category).toBe("decision");
+  });
+
+  it("strips markdown fences", () => {
+    const raw = "```json\n[{\"summary\":\"hello\",\"details\":\"\",\"category\":null,\"topics\":[],\"file_paths\":[]}]\n```";
+    const out = parseProposals(raw);
+    expect(out).toHaveLength(1);
+    expect(out[0]!.summary).toBe("hello");
+  });
+
+  it("returns [] on garbage", () => {
+    expect(parseProposals("")).toEqual([]);
+    expect(parseProposals("not json")).toEqual([]);
+    expect(parseProposals("{not array}")).toEqual([]);
+  });
+
+  it("caps at 3 proposals", () => {
+    const items = Array.from({ length: 10 }, (_, i) => ({
+      summary: `s${i}`,
+      details: "",
+      category: null,
+      topics: [],
+      file_paths: [],
+    }));
+    const raw = JSON.stringify(items);
+    expect(parseProposals(raw)).toHaveLength(3);
+  });
+
+  it("rejects items without summary", () => {
+    const raw = `[{"details":"no summary"},{"summary":"ok","details":"","category":null,"topics":[],"file_paths":[]}]`;
+    const out = parseProposals(raw);
+    expect(out).toHaveLength(1);
+    expect(out[0]!.summary).toBe("ok");
+  });
+
+  it("normalizes invalid category to null", () => {
+    const raw = `[{"summary":"x","details":"","category":"made-up","topics":[],"file_paths":[]}]`;
+    const out = parseProposals(raw);
+    expect(out[0]!.category).toBeNull();
+  });
+
+  it("uses injected ModelComplete adapter", async () => {
+    const stub = async () =>
+      `[{"summary":"stub-extracted","details":"d","category":"context","topics":["t"],"file_paths":[]}]`;
+    const ex = new MemoryExtractor(stub);
+    const out = await ex.proposeFromTurn("user msg", "assistant reply");
+    expect(out).toHaveLength(1);
+    expect(out[0]!.summary).toBe("stub-extracted");
+  });
+
+  it("collapses model errors to []", async () => {
+    const failing = async () => {
+      throw new Error("network down");
+    };
+    const ex = new MemoryExtractor(failing);
+    expect(await ex.proposeFromTurn("a", "b")).toEqual([]);
+  });
+});
+
+describe("PendingStore — persistence", () => {
+  let dir: string;
+  beforeEach(() => {
+    dir = makeTmpDir("pending");
+  });
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("survives reopen", () => {
+    const a = new PendingStore(dir);
+    a.add({
+      id: "p-1",
+      summary: "test",
+      details: "",
+      category: null,
+      topics: [],
+      file_paths: [],
+      proposed_at: new Date().toISOString(),
+      source_session_id: null,
+      source_turn_index: null,
+    });
+    const b = new PendingStore(dir);
+    expect(b.list()).toHaveLength(1);
+    expect(b.list()[0]!.id).toBe("p-1");
+  });
+
+  it("remove drops the entry", () => {
+    const s = new PendingStore(dir);
+    s.add({
+      id: "p-1",
+      summary: "x",
+      details: "",
+      category: null,
+      topics: [],
+      file_paths: [],
+      proposed_at: new Date().toISOString(),
+      source_session_id: null,
+      source_turn_index: null,
+    });
+    expect(s.remove("p-1")).toBe(true);
+    expect(s.list()).toHaveLength(0);
+  });
+});
+
+describe("MemoryManager — accept/reject pending", () => {
+  let dir: string;
+  beforeEach(() => {
+    dir = makeTmpDir("mgr");
+  });
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("addPending → accept writes a real memory and removes pending", () => {
+    const mgr = new MemoryManager(dir);
+    try {
+      const added = mgr.addPending(
+        [
+          {
+            summary: "from-extraction",
+            details: "rationale",
+            category: "decision",
+            topics: ["t1"],
+            file_paths: ["src/foo.ts"],
+          },
+        ],
+        "session-x",
+        2,
+      );
+      expect(added).toHaveLength(1);
+      expect(mgr.listPending()).toHaveLength(1);
+
+      const record = mgr.acceptPending(added[0]!.id, "project");
+      expect(record).not.toBeNull();
+      expect(record!.summary).toBe("from-extraction");
+      expect(record!.session_id).toBe("session-x");
+      expect(mgr.listPending()).toHaveLength(0);
+
+      // file_paths attached
+      const refs = mgr.listFileRefs("project", record!.id);
+      expect(refs.map((r) => r.path)).toContain("src/foo.ts");
+    } finally {
+      mgr.close();
+    }
+  });
+
+  it("rejectPending discards without writing", () => {
+    const mgr = new MemoryManager(dir);
+    try {
+      const added = mgr.addPending(
+        [
+          {
+            summary: "ignore-me",
+            details: "",
+            category: null,
+            topics: [],
+            file_paths: [],
+          },
+        ],
+        null,
+        null,
+      );
+      expect(mgr.rejectPending(added[0]!.id)).toBe(true);
+      expect(mgr.listPending()).toHaveLength(0);
+      expect(mgr.list("project").length).toBe(0);
+    } finally {
+      mgr.close();
+    }
+  });
+});