mirror of
https://github.com/langgenius/dify.git
synced 2026-08-24 04:13:41 -04:00
fix(knowledge-fs): show initial import sync time
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 5,
|
||||
"subtreeTree": "cd6c4687eaafa5e178c79aaa52c88869666f36b4",
|
||||
"subtreeTree": "3a28cc7e18419f502d1a831f63b1cd2c8e733574",
|
||||
"openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796",
|
||||
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
|
||||
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Source initial import `lastSyncedAt`
|
||||
|
||||
Date: 2026-08-21
|
||||
|
||||
## What changed
|
||||
|
||||
- The source list now derives `lastSyncedAt` from both a successful initial source import and a
|
||||
later source sync.
|
||||
- Successful Firecrawl preview selections, explicit crawl imports, online-document imports, and
|
||||
online-drive imports are treated as source content updates.
|
||||
- A preview that was never imported and an empty initial crawl remain excluded. A successful sync
|
||||
with zero new results continues to count as a completed sync.
|
||||
|
||||
## Why
|
||||
|
||||
The source-list query previously considered only workflow runs whose kind was `sync`. A newly
|
||||
created Firecrawl source can already be active and contain imported documents while its only
|
||||
completed workflow is the initial `crawl-preview` selection import. Until the first scheduled or
|
||||
manual refresh, the API therefore omitted `lastSyncedAt` and the UI displayed an em dash.
|
||||
|
||||
## Verification
|
||||
|
||||
- Focused source workflow and source handler regression: 125 tests passed.
|
||||
- Complete `@knowledge/api` suite: 416 files passed, 1 skipped; 4,624 tests passed, 3 skipped.
|
||||
- `@knowledge/api` typecheck passed.
|
||||
- Focused Biome checks and `git diff --check` passed.
|
||||
@@ -1836,7 +1836,7 @@ describe("database source-product workflow repository edge coverage", () => {
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("lists the latest successful sync timestamp for each requested source", async () => {
|
||||
it("lists the latest successful import or sync timestamp for each requested source", async () => {
|
||||
const calls: DatabaseExecuteInput[] = [];
|
||||
const database = testDatabase("postgres", async (input) => {
|
||||
calls.push(input);
|
||||
@@ -1873,14 +1873,20 @@ describe("database source-product workflow repository edge coverage", () => {
|
||||
"sync",
|
||||
"completed",
|
||||
"zero_results",
|
||||
"crawl-preview",
|
||||
"crawl-import",
|
||||
"online-document-import",
|
||||
"online-drive-import",
|
||||
sourceId,
|
||||
"source-b",
|
||||
],
|
||||
tableName: "source_workflow_runs",
|
||||
});
|
||||
expect(calls[0]?.sql).toContain('"source_id" IN ($6, $7)');
|
||||
expect(calls[0]?.sql).toContain('"source_id" IN ($10, $11)');
|
||||
expect(calls[0]?.sql).toContain('MAX("completed_at")');
|
||||
expect(calls[0]?.sql).toContain('"run_state" IN ($4, $5)');
|
||||
expect(calls[0]?.sql).toContain('"kind" IN ($6, $7, $8, $9)');
|
||||
expect(calls[0]?.sql).toContain('"run_state" = $4');
|
||||
await expect(
|
||||
repository.listLatestSyncCompletions({ knowledgeSpaceId, sourceIds: [], tenantId }),
|
||||
).resolves.toEqual([]);
|
||||
|
||||
@@ -1184,14 +1184,18 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
|
||||
"sync",
|
||||
"completed",
|
||||
"zero_results",
|
||||
"crawl-preview",
|
||||
"crawl-import",
|
||||
"online-document-import",
|
||||
"online-drive-import",
|
||||
...ids,
|
||||
];
|
||||
const placeholders = ids.map((_, index) => p(database, index + 6)).join(", ");
|
||||
const placeholders = ids.map((_, index) => p(database, index + 10)).join(", ");
|
||||
const result = await database.execute({
|
||||
maxRows: ids.length,
|
||||
operation: "select",
|
||||
params,
|
||||
sql: `SELECT ${q(database, "source_id")}, MAX(${q(database, "completed_at")}) AS ${q(database, "completed_at")} FROM ${q(database, runTable)} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "knowledge_space_id")} = ${p(database, 2)} AND ${q(database, "kind")} = ${p(database, 3)} AND ${q(database, "run_state")} IN (${p(database, 4)}, ${p(database, 5)}) AND ${q(database, "source_id")} IN (${placeholders}) AND ${q(database, "completed_at")} IS NOT NULL GROUP BY ${q(database, "source_id")};`,
|
||||
sql: `SELECT ${q(database, "source_id")}, MAX(${q(database, "completed_at")}) AS ${q(database, "completed_at")} FROM ${q(database, runTable)} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "knowledge_space_id")} = ${p(database, 2)} AND ((${q(database, "kind")} = ${p(database, 3)} AND ${q(database, "run_state")} IN (${p(database, 4)}, ${p(database, 5)})) OR (${q(database, "kind")} IN (${p(database, 6)}, ${p(database, 7)}, ${p(database, 8)}, ${p(database, 9)}) AND ${q(database, "run_state")} = ${p(database, 4)})) AND ${q(database, "source_id")} IN (${placeholders}) AND ${q(database, "completed_at")} IS NOT NULL GROUP BY ${q(database, "source_id")};`,
|
||||
tableName: runTable,
|
||||
});
|
||||
return result.rows.map(mapSyncCompletion);
|
||||
|
||||
@@ -1139,17 +1139,44 @@ describe("in-memory source product workflow repository", () => {
|
||||
).resolves.toMatchObject({ nextRunAt: "2026-03-02T00:00:00.000Z", revision: 2 });
|
||||
});
|
||||
|
||||
it("lists successful sync timestamps in the requested tenant and space", async () => {
|
||||
it("lists successful initial imports and sync timestamps in the requested tenant and space", async () => {
|
||||
const repository = createInMemorySourceProductWorkflowRepository();
|
||||
await terminalRun(repository, "zero-result-sync", "zero_results");
|
||||
const importedSourceIds = [
|
||||
["crawl-preview", "source-crawl-preview"],
|
||||
["crawl-import", "source-crawl-import"],
|
||||
["online-document-import", "source-online-document"],
|
||||
["online-drive-import", "source-online-drive"],
|
||||
] as const;
|
||||
for (const [kind, sourceId] of importedSourceIds) {
|
||||
await terminalRun(repository, `completed-${kind}`, "completed", { kind, sourceId });
|
||||
}
|
||||
await terminalRun(repository, "preview-only", "preview_ready", {
|
||||
kind: "crawl-preview",
|
||||
sourceId: "source-preview-only",
|
||||
});
|
||||
await terminalRun(repository, "empty-initial-crawl", "zero_results", {
|
||||
kind: "crawl-preview",
|
||||
sourceId: "source-empty-initial-crawl",
|
||||
});
|
||||
|
||||
await expect(
|
||||
repository.listLatestSyncCompletions({
|
||||
knowledgeSpaceId,
|
||||
sourceIds: ["missing", "source-memory", "source-memory"],
|
||||
sourceIds: [
|
||||
"missing",
|
||||
"source-memory",
|
||||
"source-memory",
|
||||
...importedSourceIds.map(([, sourceId]) => sourceId),
|
||||
"source-preview-only",
|
||||
"source-empty-initial-crawl",
|
||||
],
|
||||
tenantId,
|
||||
}),
|
||||
).resolves.toEqual([{ completedAt: createdAt, sourceId: "source-memory" }]);
|
||||
).resolves.toEqual([
|
||||
{ completedAt: createdAt, sourceId: "source-memory" },
|
||||
...importedSourceIds.map(([, sourceId]) => ({ completedAt: createdAt, sourceId })),
|
||||
]);
|
||||
await expect(
|
||||
repository.listLatestSyncCompletions({
|
||||
knowledgeSpaceId,
|
||||
@@ -1311,9 +1338,10 @@ function fence(run: SourceWorkflowRun, workerId: string) {
|
||||
async function terminalRun(
|
||||
repository: SourceProductWorkflowRepository,
|
||||
id: string,
|
||||
state: "completed" | "zero_results",
|
||||
state: "completed" | "preview_ready" | "zero_results",
|
||||
patch: Partial<NewSourceWorkflowRun> = {},
|
||||
) {
|
||||
await repository.start(runRecord(id));
|
||||
await repository.start(runRecord(id, patch));
|
||||
const claimed = requiredClaim(
|
||||
await repository.claim({
|
||||
leaseExpiresAt: "2026-03-01T01:00:00.000Z",
|
||||
|
||||
@@ -697,8 +697,14 @@ export function createInMemorySourceProductWorkflowRepository(input?: {
|
||||
if (
|
||||
run.tenantId !== tenantId ||
|
||||
run.knowledgeSpaceId !== knowledgeSpaceId ||
|
||||
run.kind !== "sync" ||
|
||||
(run.state !== "completed" && run.state !== "zero_results") ||
|
||||
!(
|
||||
(run.kind === "sync" && (run.state === "completed" || run.state === "zero_results")) ||
|
||||
(run.state === "completed" &&
|
||||
(run.kind === "crawl-preview" ||
|
||||
run.kind === "crawl-import" ||
|
||||
run.kind === "online-document-import" ||
|
||||
run.kind === "online-drive-import"))
|
||||
) ||
|
||||
!run.sourceId ||
|
||||
!run.completedAt ||
|
||||
!requestedSourceIds.has(run.sourceId)
|
||||
|
||||
@@ -407,6 +407,11 @@ export interface SourceProductWorkflowRepository {
|
||||
readonly sourceIds: readonly string[];
|
||||
readonly tenantId: string;
|
||||
}): Promise<readonly SourceWorkflowRun[]>;
|
||||
/**
|
||||
* Returns the latest successful source content update. This includes the
|
||||
* initial import as well as later sync runs so a newly imported source has a
|
||||
* meaningful `lastSyncedAt` value before its first scheduled refresh.
|
||||
*/
|
||||
listLatestSyncCompletions(input: {
|
||||
readonly knowledgeSpaceId: string;
|
||||
readonly sourceIds: readonly string[];
|
||||
|
||||
Reference in New Issue
Block a user