fix(knowledge-fs): show initial import sync time

This commit is contained in:
Jyong
2026-08-21 05:32:15 -04:00
parent 3f36ff11f7
commit 3cfb1d43ba
7 changed files with 87 additions and 12 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"schemaVersion": 5,
"subtreeTree": "cd6c4687eaafa5e178c79aaa52c88869666f36b4",
"subtreeTree": "3a28cc7e18419f502d1a831f63b1cd2c8e733574",
"openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796",
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
@@ -0,0 +1,26 @@
# Source initial import `lastSyncedAt`
Date: 2026-08-21
## What changed
- The source list now derives `lastSyncedAt` from both a successful initial source import and a
later source sync.
- Successful Firecrawl preview selections, explicit crawl imports, online-document imports, and
online-drive imports are treated as source content updates.
- A preview that was never imported and an empty initial crawl remain excluded. A successful sync
with zero new results continues to count as a completed sync.
## Why
The source-list query previously considered only workflow runs whose kind was `sync`. A newly
created Firecrawl source can already be active and contain imported documents while its only
completed workflow is the initial `crawl-preview` selection import. Until the first scheduled or
manual refresh, the API therefore omitted `lastSyncedAt` and the UI displayed an em dash.
## Verification
- Focused source workflow and source handler regression: 125 tests passed.
- Complete `@knowledge/api` suite: 416 files passed, 1 skipped; 4,624 tests passed, 3 skipped.
- `@knowledge/api` typecheck passed.
- Focused Biome checks and `git diff --check` passed.
@@ -1836,7 +1836,7 @@ describe("database source-product workflow repository edge coverage", () => {
expect(calls).toHaveLength(1);
});
it("lists the latest successful sync timestamp for each requested source", async () => {
it("lists the latest successful import or sync timestamp for each requested source", async () => {
const calls: DatabaseExecuteInput[] = [];
const database = testDatabase("postgres", async (input) => {
calls.push(input);
@@ -1873,14 +1873,20 @@ describe("database source-product workflow repository edge coverage", () => {
"sync",
"completed",
"zero_results",
"crawl-preview",
"crawl-import",
"online-document-import",
"online-drive-import",
sourceId,
"source-b",
],
tableName: "source_workflow_runs",
});
expect(calls[0]?.sql).toContain('"source_id" IN ($6, $7)');
expect(calls[0]?.sql).toContain('"source_id" IN ($10, $11)');
expect(calls[0]?.sql).toContain('MAX("completed_at")');
expect(calls[0]?.sql).toContain('"run_state" IN ($4, $5)');
expect(calls[0]?.sql).toContain('"kind" IN ($6, $7, $8, $9)');
expect(calls[0]?.sql).toContain('"run_state" = $4');
await expect(
repository.listLatestSyncCompletions({ knowledgeSpaceId, sourceIds: [], tenantId }),
).resolves.toEqual([]);
@@ -1184,14 +1184,18 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
"sync",
"completed",
"zero_results",
"crawl-preview",
"crawl-import",
"online-document-import",
"online-drive-import",
...ids,
];
const placeholders = ids.map((_, index) => p(database, index + 6)).join(", ");
const placeholders = ids.map((_, index) => p(database, index + 10)).join(", ");
const result = await database.execute({
maxRows: ids.length,
operation: "select",
params,
sql: `SELECT ${q(database, "source_id")}, MAX(${q(database, "completed_at")}) AS ${q(database, "completed_at")} FROM ${q(database, runTable)} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "knowledge_space_id")} = ${p(database, 2)} AND ${q(database, "kind")} = ${p(database, 3)} AND ${q(database, "run_state")} IN (${p(database, 4)}, ${p(database, 5)}) AND ${q(database, "source_id")} IN (${placeholders}) AND ${q(database, "completed_at")} IS NOT NULL GROUP BY ${q(database, "source_id")};`,
sql: `SELECT ${q(database, "source_id")}, MAX(${q(database, "completed_at")}) AS ${q(database, "completed_at")} FROM ${q(database, runTable)} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "knowledge_space_id")} = ${p(database, 2)} AND ((${q(database, "kind")} = ${p(database, 3)} AND ${q(database, "run_state")} IN (${p(database, 4)}, ${p(database, 5)})) OR (${q(database, "kind")} IN (${p(database, 6)}, ${p(database, 7)}, ${p(database, 8)}, ${p(database, 9)}) AND ${q(database, "run_state")} = ${p(database, 4)})) AND ${q(database, "source_id")} IN (${placeholders}) AND ${q(database, "completed_at")} IS NOT NULL GROUP BY ${q(database, "source_id")};`,
tableName: runTable,
});
return result.rows.map(mapSyncCompletion);
@@ -1139,17 +1139,44 @@ describe("in-memory source product workflow repository", () => {
).resolves.toMatchObject({ nextRunAt: "2026-03-02T00:00:00.000Z", revision: 2 });
});
it("lists successful sync timestamps in the requested tenant and space", async () => {
it("lists successful initial imports and sync timestamps in the requested tenant and space", async () => {
const repository = createInMemorySourceProductWorkflowRepository();
await terminalRun(repository, "zero-result-sync", "zero_results");
const importedSourceIds = [
["crawl-preview", "source-crawl-preview"],
["crawl-import", "source-crawl-import"],
["online-document-import", "source-online-document"],
["online-drive-import", "source-online-drive"],
] as const;
for (const [kind, sourceId] of importedSourceIds) {
await terminalRun(repository, `completed-${kind}`, "completed", { kind, sourceId });
}
await terminalRun(repository, "preview-only", "preview_ready", {
kind: "crawl-preview",
sourceId: "source-preview-only",
});
await terminalRun(repository, "empty-initial-crawl", "zero_results", {
kind: "crawl-preview",
sourceId: "source-empty-initial-crawl",
});
await expect(
repository.listLatestSyncCompletions({
knowledgeSpaceId,
sourceIds: ["missing", "source-memory", "source-memory"],
sourceIds: [
"missing",
"source-memory",
"source-memory",
...importedSourceIds.map(([, sourceId]) => sourceId),
"source-preview-only",
"source-empty-initial-crawl",
],
tenantId,
}),
).resolves.toEqual([{ completedAt: createdAt, sourceId: "source-memory" }]);
).resolves.toEqual([
{ completedAt: createdAt, sourceId: "source-memory" },
...importedSourceIds.map(([, sourceId]) => ({ completedAt: createdAt, sourceId })),
]);
await expect(
repository.listLatestSyncCompletions({
knowledgeSpaceId,
@@ -1311,9 +1338,10 @@ function fence(run: SourceWorkflowRun, workerId: string) {
async function terminalRun(
repository: SourceProductWorkflowRepository,
id: string,
state: "completed" | "zero_results",
state: "completed" | "preview_ready" | "zero_results",
patch: Partial<NewSourceWorkflowRun> = {},
) {
await repository.start(runRecord(id));
await repository.start(runRecord(id, patch));
const claimed = requiredClaim(
await repository.claim({
leaseExpiresAt: "2026-03-01T01:00:00.000Z",
@@ -697,8 +697,14 @@ export function createInMemorySourceProductWorkflowRepository(input?: {
if (
run.tenantId !== tenantId ||
run.knowledgeSpaceId !== knowledgeSpaceId ||
run.kind !== "sync" ||
(run.state !== "completed" && run.state !== "zero_results") ||
!(
(run.kind === "sync" && (run.state === "completed" || run.state === "zero_results")) ||
(run.state === "completed" &&
(run.kind === "crawl-preview" ||
run.kind === "crawl-import" ||
run.kind === "online-document-import" ||
run.kind === "online-drive-import"))
) ||
!run.sourceId ||
!run.completedAt ||
!requestedSourceIds.has(run.sourceId)
@@ -407,6 +407,11 @@ export interface SourceProductWorkflowRepository {
readonly sourceIds: readonly string[];
readonly tenantId: string;
}): Promise<readonly SourceWorkflowRun[]>;
/**
* Returns the latest successful source content update. This includes the
* initial import as well as later sync runs so a newly imported source has a
* meaningful `lastSyncedAt` value before its first scheduled refresh.
*/
listLatestSyncCompletions(input: {
readonly knowledgeSpaceId: string;
readonly sourceIds: readonly string[];