From 3cfb1d43ba803963a46c7e3bb7d5750418ba70e9 Mon Sep 17 00:00:00 2001 From: Jyong Date: Fri, 21 Aug 2026 05:32:15 -0400 Subject: [PATCH] fix(knowledge-fs): show initial import sync time --- api/knowledge-fs-contract.lock.json | 2 +- ...21-source-initial-import-last-synced-at.md | 26 +++++++++++++ ...oduct-workflow-database-repository.test.ts | 10 ++++- ...ce-product-workflow-database-repository.ts | 8 +++- ...product-workflow-memory-repository.test.ts | 38 ++++++++++++++++--- ...urce-product-workflow-memory-repository.ts | 10 ++++- .../api/src/source-product-workflow.ts | 5 +++ 7 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 knowledge-fs/.harness/changes/2026-08-21-source-initial-import-last-synced-at.md diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 1ed65e156ab..90827f8c968 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "cd6c4687eaafa5e178c79aaa52c88869666f36b4", + "subtreeTree": "3a28cc7e18419f502d1a831f63b1cd2c8e733574", "openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796", "capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", diff --git a/knowledge-fs/.harness/changes/2026-08-21-source-initial-import-last-synced-at.md b/knowledge-fs/.harness/changes/2026-08-21-source-initial-import-last-synced-at.md new file mode 100644 index 00000000000..5ff3ef3a030 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-08-21-source-initial-import-last-synced-at.md @@ -0,0 +1,26 @@ +# Source initial import `lastSyncedAt` + +Date: 2026-08-21 + +## What changed + +- The source list now derives `lastSyncedAt` from both a successful initial source import and a + later source sync. +- Successful Firecrawl preview selections, explicit crawl imports, online-document imports, and + online-drive imports are treated as source content updates. +- A preview that was never imported and an empty initial crawl remain excluded. A successful sync + with zero new results continues to count as a completed sync. + +## Why + +The source-list query previously considered only workflow runs whose kind was `sync`. A newly +created Firecrawl source can already be active and contain imported documents while its only +completed workflow is the initial `crawl-preview` selection import. Until the first scheduled or +manual refresh, the API therefore omitted `lastSyncedAt` and the UI displayed an em dash. + +## Verification + +- Focused source workflow and source handler regression: 125 tests passed. +- Complete `@knowledge/api` suite: 416 files passed, 1 skipped; 4,624 tests passed, 3 skipped. +- `@knowledge/api` typecheck passed. +- Focused Biome checks and `git diff --check` passed. diff --git a/knowledge-fs/packages/api/src/source-product-workflow-database-repository.test.ts b/knowledge-fs/packages/api/src/source-product-workflow-database-repository.test.ts index 0be00154e3a..b71f260ac50 100644 --- a/knowledge-fs/packages/api/src/source-product-workflow-database-repository.test.ts +++ b/knowledge-fs/packages/api/src/source-product-workflow-database-repository.test.ts @@ -1836,7 +1836,7 @@ describe("database source-product workflow repository edge coverage", () => { expect(calls).toHaveLength(1); }); - it("lists the latest successful sync timestamp for each requested source", async () => { + it("lists the latest successful import or sync timestamp for each requested source", async () => { const calls: DatabaseExecuteInput[] = []; const database = testDatabase("postgres", async (input) => { calls.push(input); @@ -1873,14 +1873,20 @@ describe("database source-product workflow repository edge coverage", () => { "sync", "completed", "zero_results", + "crawl-preview", + "crawl-import", + "online-document-import", + "online-drive-import", sourceId, "source-b", ], tableName: "source_workflow_runs", }); - expect(calls[0]?.sql).toContain('"source_id" IN ($6, $7)'); + expect(calls[0]?.sql).toContain('"source_id" IN ($10, $11)'); expect(calls[0]?.sql).toContain('MAX("completed_at")'); expect(calls[0]?.sql).toContain('"run_state" IN ($4, $5)'); + expect(calls[0]?.sql).toContain('"kind" IN ($6, $7, $8, $9)'); + expect(calls[0]?.sql).toContain('"run_state" = $4'); await expect( repository.listLatestSyncCompletions({ knowledgeSpaceId, sourceIds: [], tenantId }), ).resolves.toEqual([]); diff --git a/knowledge-fs/packages/api/src/source-product-workflow-database-repository.ts b/knowledge-fs/packages/api/src/source-product-workflow-database-repository.ts index ea63329c806..c8d708a43d2 100644 --- a/knowledge-fs/packages/api/src/source-product-workflow-database-repository.ts +++ b/knowledge-fs/packages/api/src/source-product-workflow-database-repository.ts @@ -1184,14 +1184,18 @@ export function createDatabaseSourceProductWorkflowRepository(input: { "sync", "completed", "zero_results", + "crawl-preview", + "crawl-import", + "online-document-import", + "online-drive-import", ...ids, ]; - const placeholders = ids.map((_, index) => p(database, index + 6)).join(", "); + const placeholders = ids.map((_, index) => p(database, index + 10)).join(", "); const result = await database.execute({ maxRows: ids.length, operation: "select", params, - sql: `SELECT ${q(database, "source_id")}, MAX(${q(database, "completed_at")}) AS ${q(database, "completed_at")} FROM ${q(database, runTable)} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "knowledge_space_id")} = ${p(database, 2)} AND ${q(database, "kind")} = ${p(database, 3)} AND ${q(database, "run_state")} IN (${p(database, 4)}, ${p(database, 5)}) AND ${q(database, "source_id")} IN (${placeholders}) AND ${q(database, "completed_at")} IS NOT NULL GROUP BY ${q(database, "source_id")};`, + sql: `SELECT ${q(database, "source_id")}, MAX(${q(database, "completed_at")}) AS ${q(database, "completed_at")} FROM ${q(database, runTable)} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "knowledge_space_id")} = ${p(database, 2)} AND ((${q(database, "kind")} = ${p(database, 3)} AND ${q(database, "run_state")} IN (${p(database, 4)}, ${p(database, 5)})) OR (${q(database, "kind")} IN (${p(database, 6)}, ${p(database, 7)}, ${p(database, 8)}, ${p(database, 9)}) AND ${q(database, "run_state")} = ${p(database, 4)})) AND ${q(database, "source_id")} IN (${placeholders}) AND ${q(database, "completed_at")} IS NOT NULL GROUP BY ${q(database, "source_id")};`, tableName: runTable, }); return result.rows.map(mapSyncCompletion); diff --git a/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.test.ts b/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.test.ts index a9dc198052a..9eb50b65e95 100644 --- a/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.test.ts +++ b/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.test.ts @@ -1139,17 +1139,44 @@ describe("in-memory source product workflow repository", () => { ).resolves.toMatchObject({ nextRunAt: "2026-03-02T00:00:00.000Z", revision: 2 }); }); - it("lists successful sync timestamps in the requested tenant and space", async () => { + it("lists successful initial imports and sync timestamps in the requested tenant and space", async () => { const repository = createInMemorySourceProductWorkflowRepository(); await terminalRun(repository, "zero-result-sync", "zero_results"); + const importedSourceIds = [ + ["crawl-preview", "source-crawl-preview"], + ["crawl-import", "source-crawl-import"], + ["online-document-import", "source-online-document"], + ["online-drive-import", "source-online-drive"], + ] as const; + for (const [kind, sourceId] of importedSourceIds) { + await terminalRun(repository, `completed-${kind}`, "completed", { kind, sourceId }); + } + await terminalRun(repository, "preview-only", "preview_ready", { + kind: "crawl-preview", + sourceId: "source-preview-only", + }); + await terminalRun(repository, "empty-initial-crawl", "zero_results", { + kind: "crawl-preview", + sourceId: "source-empty-initial-crawl", + }); await expect( repository.listLatestSyncCompletions({ knowledgeSpaceId, - sourceIds: ["missing", "source-memory", "source-memory"], + sourceIds: [ + "missing", + "source-memory", + "source-memory", + ...importedSourceIds.map(([, sourceId]) => sourceId), + "source-preview-only", + "source-empty-initial-crawl", + ], tenantId, }), - ).resolves.toEqual([{ completedAt: createdAt, sourceId: "source-memory" }]); + ).resolves.toEqual([ + { completedAt: createdAt, sourceId: "source-memory" }, + ...importedSourceIds.map(([, sourceId]) => ({ completedAt: createdAt, sourceId })), + ]); await expect( repository.listLatestSyncCompletions({ knowledgeSpaceId, @@ -1311,9 +1338,10 @@ function fence(run: SourceWorkflowRun, workerId: string) { async function terminalRun( repository: SourceProductWorkflowRepository, id: string, - state: "completed" | "zero_results", + state: "completed" | "preview_ready" | "zero_results", + patch: Partial = {}, ) { - await repository.start(runRecord(id)); + await repository.start(runRecord(id, patch)); const claimed = requiredClaim( await repository.claim({ leaseExpiresAt: "2026-03-01T01:00:00.000Z", diff --git a/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.ts b/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.ts index e488d760ef1..fa983c685cf 100644 --- a/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.ts +++ b/knowledge-fs/packages/api/src/source-product-workflow-memory-repository.ts @@ -697,8 +697,14 @@ export function createInMemorySourceProductWorkflowRepository(input?: { if ( run.tenantId !== tenantId || run.knowledgeSpaceId !== knowledgeSpaceId || - run.kind !== "sync" || - (run.state !== "completed" && run.state !== "zero_results") || + !( + (run.kind === "sync" && (run.state === "completed" || run.state === "zero_results")) || + (run.state === "completed" && + (run.kind === "crawl-preview" || + run.kind === "crawl-import" || + run.kind === "online-document-import" || + run.kind === "online-drive-import")) + ) || !run.sourceId || !run.completedAt || !requestedSourceIds.has(run.sourceId) diff --git a/knowledge-fs/packages/api/src/source-product-workflow.ts b/knowledge-fs/packages/api/src/source-product-workflow.ts index 689ce88de6f..ae2b51b614e 100644 --- a/knowledge-fs/packages/api/src/source-product-workflow.ts +++ b/knowledge-fs/packages/api/src/source-product-workflow.ts @@ -407,6 +407,11 @@ export interface SourceProductWorkflowRepository { readonly sourceIds: readonly string[]; readonly tenantId: string; }): Promise; + /** + * Returns the latest successful source content update. This includes the + * initial import as well as later sync runs so a newly imported source has a + * meaningful `lastSyncedAt` value before its first scheduled refresh. + */ listLatestSyncCompletions(input: { readonly knowledgeSpaceId: string; readonly sourceIds: readonly string[];